2026-07-10T21:29:20Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-ad1d1c21"}, "namespace": "ai-tenant-e2e-ait-ad1d1c21", "name": "odh-trusted-ca-bundle", "reconcileID": "c2e87da1-be34-4346-9086-d35b8920c4e9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-ad1d1c21", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:29:20Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-ad1d1c21"}, "namespace": "ai-tenant-e2e-ait-ad1d1c21", "name": "odh-trusted-ca-bundle", "reconcileID": "c2e87da1-be34-4346-9086-d35b8920c4e9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-ad1d1c21"}
2026-07-10T21:29:20Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-ad1d1c21"}, "namespace": "ai-tenant-e2e-ait-ad1d1c21", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "95fc2d28-c403-4c5a-a8d5-5cb3ba3951b0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-ad1d1c21"}
2026-07-10T21:29:20Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-ad1d1c21","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-ad1d1c21", "reconcileID": "a3ec8c65-35d7-4e57-950b-25dd48b76202", "Gateway": "e2e-ait-ad1d1c21", "namespace": "openshift-ingress"}
2026-07-10T21:29:21Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "c828bfe3-7e3a-4e5c-8d52-35999f12615c", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"}
2026-07-10T21:29:23Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-ad1d1c21","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-ad1d1c21", "reconcileID": "7988cf96-ae54-43ad-9284-07c2a702fe19", "Gateway": "e2e-ait-ad1d1c21", "namespace": "openshift-ingress"}
2026-07-10T21:29:32Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-ad1d1c21"}, "namespace": "ai-tenant-e2e-ait-ad1d1c21", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0610a609-3cf5-48b7-b0ac-5f7d6865493c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-ad1d1c21"}
2026-07-10T21:29:32Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-ad1d1c21"}, "namespace": "ai-tenant-e2e-ait-ad1d1c21", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0610a609-3cf5-48b7-b0ac-5f7d6865493c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-ad1d1c21", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:29:32Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-ad1d1c21"}, "namespace": "ai-tenant-e2e-ait-ad1d1c21", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0610a609-3cf5-48b7-b0ac-5f7d6865493c", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ait-ad1d1c21 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T21:29:32Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-ad1d1c21"}, "namespace": "ai-tenant-e2e-ait-ad1d1c21", "name": "odh-trusted-ca-bundle", "reconcileID": "66a6fa11-40cd-48be-99b9-d67bcf2c317a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-ad1d1c21"}
2026-07-10T21:29:32Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-ad1d1c21"}, "namespace": "ai-tenant-e2e-ait-ad1d1c21", "name": "odh-trusted-ca-bundle", "reconcileID": "66a6fa11-40cd-48be-99b9-d67bcf2c317a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-ad1d1c21"}
2026-07-10T21:29:32Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-ad1d1c21"}, "namespace": "ai-tenant-e2e-ait-ad1d1c21", "name": "openshift-service-ca.crt", "reconcileID": "13b6ab97-59fb-4ad8-81aa-bd6d2114ecd5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-ad1d1c21"}
2026-07-10T21:29:32Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-ad1d1c21"}, "namespace": "ai-tenant-e2e-ait-ad1d1c21", "name": "openshift-service-ca.crt", "reconcileID": "13b6ab97-59fb-4ad8-81aa-bd6d2114ecd5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-ad1d1c21"}
2026-07-10T21:29:32Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-ad1d1c21"}, "namespace": "ai-tenant-e2e-ait-ad1d1c21", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d0852991-9f11-4cf0-a734-43009bdf3213", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-ad1d1c21"}
2026-07-10T21:29:32Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-ad1d1c21"}, "namespace": "ai-tenant-e2e-ait-ad1d1c21", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d0852991-9f11-4cf0-a734-43009bdf3213", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-ad1d1c21"}
2026-07-10T21:29:32Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-78079c65","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-78079c65", "reconcileID": "6ea2bb8a-238d-4761-8ad8-16064ed2b284", "Gateway": "e2e-derive-78079c65", "namespace": "openshift-ingress"}
2026-07-10T21:29:32Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-78079c65","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-78079c65", "reconcileID": "eb6ed07e-dcf0-4647-a559-d7501b2d9b3c", "Gateway": "e2e-derive-78079c65", "namespace": "openshift-ingress"}
2026-07-10T21:29:32Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-78079c65","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-78079c65", "reconcileID": "09d87e33-bad3-45f2-acb8-434173f5b5a1", "Gateway": "e2e-derive-78079c65", "namespace": "openshift-ingress"}
2026-07-10T21:29:32Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-78079c65","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-78079c65", "reconcileID": "4853a734-b757-4a59-b52f-1f971a491ee8", "Gateway": "e2e-derive-78079c65", "namespace": "openshift-ingress"}
2026-07-10T21:29:32Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-78079c65","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-78079c65", "reconcileID": "ab05bac2-c44a-4b96-a131-3069fc6ba5e2", "Gateway": "e2e-derive-78079c65", "namespace": "openshift-ingress"}
2026-07-10T21:29:33Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-derive-78079c65"}, "namespace": "ai-tenant-e2e-derive-78079c65", "name": "openshift-service-ca.crt", "reconcileID": "9c38e132-f347-4515-8c52-dba3ea1eefb3", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-derive-78079c65"}
2026-07-10T21:29:33Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-derive-78079c65"}, "namespace": "ai-tenant-e2e-derive-78079c65", "name": "openshift-service-ca.crt", "reconcileID": "649d96d3-b89a-4ebc-b5ef-aef57ce92c1e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-derive-78079c65", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:29:33Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-derive-78079c65"}, "namespace": "ai-tenant-e2e-derive-78079c65", "name": "openshift-service-ca.crt", "reconcileID": "649d96d3-b89a-4ebc-b5ef-aef57ce92c1e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-derive-78079c65"}
2026-07-10T21:29:33Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-derive-78079c65"}, "namespace": "ai-tenant-e2e-derive-78079c65", "name": "odh-trusted-ca-bundle", "reconcileID": "85d1fb53-ae95-45d3-9d1d-04171478ce87", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-derive-78079c65"}
2026-07-10T21:29:33Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-derive-78079c65"}, "namespace": "ai-tenant-e2e-derive-78079c65", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c7f61c73-23e3-4ce2-b96e-c5b369d3a65f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-derive-78079c65"}
2026-07-10T21:29:33Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-derive-78079c65"}, "namespace": "ai-tenant-e2e-derive-78079c65", "name": "openshift-service-ca.crt", "reconcileID": "37ef3567-247f-4c8f-bef7-5a2e1e0927c2", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-derive-78079c65"}
2026-07-10T21:29:33Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-derive-78079c65"}, "namespace": "ai-tenant-e2e-derive-78079c65", "name": "odh-trusted-ca-bundle", "reconcileID": "a3b9991d-afa2-48ab-8298-10230b64be87", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-derive-78079c65", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:29:33Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-derive-78079c65"}, "namespace": "ai-tenant-e2e-derive-78079c65", "name": "odh-trusted-ca-bundle", "reconcileID": "a3b9991d-afa2-48ab-8298-10230b64be87", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-derive-78079c65"}
2026-07-10T21:29:33Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-derive-78079c65"}, "namespace": "ai-tenant-e2e-derive-78079c65", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f5b138d2-d6af-479a-8f80-7ccd93eb93ef", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-derive-78079c65"}
2026-07-10T21:29:33Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-78079c65","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-78079c65", "reconcileID": "69b34788-46b0-48be-9578-d49d88e97f58", "Gateway": "e2e-derive-78079c65", "namespace": "openshift-ingress"}
2026-07-10T21:29:34Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "ad95967b-3a02-4ee7-a6a6-9d5d4be0b6ae", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"}
2026-07-10T21:29:36Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-78079c65","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-78079c65", "reconcileID": "16793ad2-4479-4d92-ac54-3402ed76fa01", "Gateway": "e2e-derive-78079c65", "namespace": "openshift-ingress"}
2026-07-10T21:29:43Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-derive-78079c65"}, "namespace": "ai-tenant-e2e-derive-78079c65", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "13574234-dcfd-4138-bae9-4a5e1811ba44", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-derive-78079c65"}
2026-07-10T21:29:43Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-derive-78079c65"}, "namespace": "ai-tenant-e2e-derive-78079c65", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "13574234-dcfd-4138-bae9-4a5e1811ba44", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-derive-78079c65", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:29:43Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-derive-78079c65"}, "namespace": "ai-tenant-e2e-derive-78079c65", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "13574234-dcfd-4138-bae9-4a5e1811ba44", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-derive-78079c65 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T21:29:43Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-derive-78079c65"}, "namespace": "ai-tenant-e2e-derive-78079c65", "name": "odh-trusted-ca-bundle", "reconcileID": "2775b1dc-9f56-438e-93c4-74d7eb3c8deb", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-derive-78079c65"}
2026-07-10T21:29:43Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-derive-78079c65"}, "namespace": "ai-tenant-e2e-derive-78079c65", "name": "odh-trusted-ca-bundle", "reconcileID": "2775b1dc-9f56-438e-93c4-74d7eb3c8deb", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-derive-78079c65"}
2026-07-10T21:29:43Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-derive-78079c65"}, "namespace": "ai-tenant-e2e-derive-78079c65", "name": "openshift-service-ca.crt", "reconcileID": "e0e3d45a-3e8a-4da3-84a8-adf406d4f764", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-derive-78079c65"}
2026-07-10T21:29:43Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-derive-78079c65"}, "namespace": "ai-tenant-e2e-derive-78079c65", "name": "openshift-service-ca.crt", "reconcileID": "e0e3d45a-3e8a-4da3-84a8-adf406d4f764", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-derive-78079c65"}
2026-07-10T21:29:43Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-derive-78079c65"}, "namespace": "ai-tenant-e2e-derive-78079c65", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "72078731-40f2-428c-86f2-2a564403fbea", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-derive-78079c65"}
2026-07-10T21:29:43Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-derive-78079c65"}, "namespace": "ai-tenant-e2e-derive-78079c65", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "72078731-40f2-428c-86f2-2a564403fbea", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-derive-78079c65"}
2026-07-10T21:29:44Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-ba68cd97"}, "namespace": "ai-tenant-e2e-mt-ba68cd97", "name": "openshift-service-ca.crt", "reconcileID": "3af5a475-b90e-40c5-a705-cde15c9dc969", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-ba68cd97"}
2026-07-10T21:29:44Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-ba68cd97"}, "namespace": "ai-tenant-e2e-mt-ba68cd97", "name": "odh-trusted-ca-bundle", "reconcileID": "6d6791ae-5aa9-4148-95c4-d280bf462b97", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-ba68cd97"}
2026-07-10T21:29:44Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-ba68cd97"}, "namespace": "ai-tenant-e2e-mt-ba68cd97", "name": "openshift-service-ca.crt", "reconcileID": "87b90503-f4f8-4014-92eb-34ec131057e8", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-ba68cd97", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:29:44Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-ba68cd97"}, "namespace": "ai-tenant-e2e-mt-ba68cd97", "name": "openshift-service-ca.crt", "reconcileID": "87b90503-f4f8-4014-92eb-34ec131057e8", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-ba68cd97"}
2026-07-10T21:29:44Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-ba68cd97"}, "namespace": "ai-tenant-e2e-mt-ba68cd97", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e38a2a2e-9493-4eea-9b8c-bcb55100d6ec", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-ba68cd97"}
2026-07-10T21:29:44Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-ba68cd97"}, "namespace": "ai-tenant-e2e-mt-ba68cd97", "name": "openshift-service-ca.crt", "reconcileID": "50de287b-577a-4ff3-b00e-6668d5759b81", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-ba68cd97"}
2026-07-10T21:29:44Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-ba68cd97"}, "namespace": "ai-tenant-e2e-mt-ba68cd97", "name": "odh-trusted-ca-bundle", "reconcileID": "85757635-f9c3-48cf-bc27-ca938fd1964c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-ba68cd97", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:29:44Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-ba68cd97"}, "namespace": "ai-tenant-e2e-mt-ba68cd97", "name": "odh-trusted-ca-bundle", "reconcileID": "85757635-f9c3-48cf-bc27-ca938fd1964c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-ba68cd97"}
2026-07-10T21:29:44Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-ba68cd97"}, "namespace": "ai-tenant-e2e-mt-ba68cd97", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "940e554f-7ded-40d2-9edc-d8f42186861a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-ba68cd97"}
2026-07-10T21:29:51Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-ba68cd97"}, "namespace": "ai-tenant-e2e-mt-ba68cd97", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "808adfad-ffa6-4fbf-9b0b-785daf696c5f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-ba68cd97"}
2026-07-10T21:29:51Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-ba68cd97"}, "namespace": "ai-tenant-e2e-mt-ba68cd97", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "808adfad-ffa6-4fbf-9b0b-785daf696c5f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-ba68cd97", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:29:51Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-ba68cd97"}, "namespace": "ai-tenant-e2e-mt-ba68cd97", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "808adfad-ffa6-4fbf-9b0b-785daf696c5f", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-ba68cd97 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T21:29:51Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-ba68cd97"}, "namespace": "ai-tenant-e2e-mt-ba68cd97", "name": "odh-trusted-ca-bundle", "reconcileID": "f0e5d223-5550-48b9-9d5c-5335916a516f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-ba68cd97"}
2026-07-10T21:29:51Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-ba68cd97"}, "namespace": "ai-tenant-e2e-mt-ba68cd97", "name": "odh-trusted-ca-bundle", "reconcileID": "f0e5d223-5550-48b9-9d5c-5335916a516f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-ba68cd97"}
2026-07-10T21:29:51Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-ba68cd97"}, "namespace": "ai-tenant-e2e-mt-ba68cd97", "name": "openshift-service-ca.crt", "reconcileID": "83028e07-b592-4bed-93ff-5faf25227c95", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-ba68cd97"}
2026-07-10T21:29:51Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-ba68cd97"}, "namespace": "ai-tenant-e2e-mt-ba68cd97", "name": "openshift-service-ca.crt", "reconcileID": "83028e07-b592-4bed-93ff-5faf25227c95", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-ba68cd97"}
2026-07-10T21:29:51Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-ba68cd97"}, "namespace": "ai-tenant-e2e-mt-ba68cd97", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6640cb80-81ed-49d2-af12-ebd20ce8a8ab", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-ba68cd97"}
2026-07-10T21:29:51Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-ba68cd97"}, "namespace": "ai-tenant-e2e-mt-ba68cd97", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6640cb80-81ed-49d2-af12-ebd20ce8a8ab", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-ba68cd97"}
2026-07-10T21:29:52Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-75b7967b"}, "namespace": "ai-tenant-e2e-mt-75b7967b", "name": "openshift-service-ca.crt", "reconcileID": "e2e14006-ac0e-4f88-9388-e7d334cd82bf", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-75b7967b"}
2026-07-10T21:29:52Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-75b7967b"}, "namespace": "ai-tenant-e2e-mt-75b7967b", "name": "odh-trusted-ca-bundle", "reconcileID": "504c297b-0330-46d4-81d7-d1b02586d5e0", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-75b7967b"}
2026-07-10T21:29:52Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-75b7967b"}, "namespace": "ai-tenant-e2e-mt-75b7967b", "name": "openshift-service-ca.crt", "reconcileID": "157327e5-1523-4822-86a6-aa13ec282e38", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-75b7967b", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:29:52Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-75b7967b"}, "namespace": "ai-tenant-e2e-mt-75b7967b", "name": "openshift-service-ca.crt", "reconcileID": "157327e5-1523-4822-86a6-aa13ec282e38", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-75b7967b"}
2026-07-10T21:29:52Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-75b7967b"}, "namespace": "ai-tenant-e2e-mt-75b7967b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "03b1b538-0f98-4fbf-84ac-fa569e36fb6f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-75b7967b"}
2026-07-10T21:29:52Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-75b7967b"}, "namespace": "ai-tenant-e2e-mt-75b7967b", "name": "openshift-service-ca.crt", "reconcileID": "b0f1b506-a0d6-4773-b9d0-9bd78dc1754d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-75b7967b"}
2026-07-10T21:29:52Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-75b7967b"}, "namespace": "ai-tenant-e2e-mt-75b7967b", "name": "odh-trusted-ca-bundle", "reconcileID": "d0dd30db-91b6-4e1d-99f0-b9fc4cff5d52", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-75b7967b", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:29:52Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-75b7967b"}, "namespace": "ai-tenant-e2e-mt-75b7967b", "name": "odh-trusted-ca-bundle", "reconcileID": "d0dd30db-91b6-4e1d-99f0-b9fc4cff5d52", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-75b7967b"}
2026-07-10T21:29:52Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-75b7967b"}, "namespace": "ai-tenant-e2e-mt-75b7967b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f283429c-4c14-4ae5-81d2-a92f6a20463f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-75b7967b"}
2026-07-10T21:30:24Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-75b7967b"}, "namespace": "ai-tenant-e2e-mt-75b7967b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bc6d4fb3-26de-4511-82a4-a8e4402c267c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-75b7967b"}
2026-07-10T21:30:24Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-75b7967b"}, "namespace": "ai-tenant-e2e-mt-75b7967b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bc6d4fb3-26de-4511-82a4-a8e4402c267c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-75b7967b", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:30:24Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-75b7967b"}, "namespace": "ai-tenant-e2e-mt-75b7967b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bc6d4fb3-26de-4511-82a4-a8e4402c267c", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-75b7967b because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T21:30:24Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-75b7967b"}, "namespace": "ai-tenant-e2e-mt-75b7967b", "name": "odh-trusted-ca-bundle", "reconcileID": "3f89104a-d997-4efe-ba1a-7917a0a23f33", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-75b7967b"}
2026-07-10T21:30:24Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-75b7967b"}, "namespace": "ai-tenant-e2e-mt-75b7967b", "name": "odh-trusted-ca-bundle", "reconcileID": "3f89104a-d997-4efe-ba1a-7917a0a23f33", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-75b7967b"}
2026-07-10T21:30:24Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-75b7967b"}, "namespace": "ai-tenant-e2e-mt-75b7967b", "name": "openshift-service-ca.crt", "reconcileID": "313bf898-6ffc-45a2-af0e-4a9b5174556b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-75b7967b"}
2026-07-10T21:30:24Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-75b7967b"}, "namespace": "ai-tenant-e2e-mt-75b7967b", "name": "openshift-service-ca.crt", "reconcileID": "313bf898-6ffc-45a2-af0e-4a9b5174556b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-75b7967b"}
2026-07-10T21:30:24Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-75b7967b"}, "namespace": "ai-tenant-e2e-mt-75b7967b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "83a385ec-c9f4-4590-965b-d521d1b559c2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-75b7967b"}
2026-07-10T21:30:24Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-75b7967b"}, "namespace": "ai-tenant-e2e-mt-75b7967b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "83a385ec-c9f4-4590-965b-d521d1b559c2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-75b7967b"}
2026-07-10T21:30:24Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-da109fcf"}, "namespace": "e2e-unlabeled-da109fcf", "name": "openshift-service-ca.crt", "reconcileID": "8bedc246-2450-43da-a8e0-22bccacf83ef", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-da109fcf"}
2026-07-10T21:30:24Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-da109fcf"}, "namespace": "e2e-unlabeled-da109fcf", "name": "odh-trusted-ca-bundle", "reconcileID": "291d31b3-20f4-4b21-8a1c-dd9ce11167ad", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-da109fcf"}
2026-07-10T21:30:24Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-da109fcf"}, "namespace": "e2e-unlabeled-da109fcf", "name": "openshift-service-ca.crt", "reconcileID": "aa557b4e-9243-4841-99ad-5988f5635104", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-da109fcf", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:30:24Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-da109fcf"}, "namespace": "e2e-unlabeled-da109fcf", "name": "openshift-service-ca.crt", "reconcileID": "aa557b4e-9243-4841-99ad-5988f5635104", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-da109fcf"}
2026-07-10T21:30:24Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-da109fcf"}, "namespace": "e2e-unlabeled-da109fcf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "70116136-43a1-46a3-9a04-29db9402ef1f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-da109fcf"}
2026-07-10T21:30:24Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-da109fcf"}, "namespace": "e2e-unlabeled-da109fcf", "name": "openshift-service-ca.crt", "reconcileID": "d8210316-eb9b-45b9-a0ec-89185c255190", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-da109fcf"}
2026-07-10T21:30:24Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-da109fcf"}, "namespace": "e2e-unlabeled-da109fcf", "name": "odh-trusted-ca-bundle", "reconcileID": "23f8485d-59da-494c-99dd-be609b06222e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-da109fcf", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:30:24Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-da109fcf"}, "namespace": "e2e-unlabeled-da109fcf", "name": "odh-trusted-ca-bundle", "reconcileID": "23f8485d-59da-494c-99dd-be609b06222e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-da109fcf"}
2026-07-10T21:30:24Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-da109fcf"}, "namespace": "e2e-unlabeled-da109fcf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "29f70e85-6e85-4d20-adf5-d7b2aa48b81d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-da109fcf"}
2026-07-10T21:30:46Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-da109fcf"}, "namespace": "e2e-unlabeled-da109fcf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ed9d9587-e751-423e-9859-e3ed28cd4f90", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-da109fcf"}
2026-07-10T21:30:46Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-da109fcf"}, "namespace": "e2e-unlabeled-da109fcf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ed9d9587-e751-423e-9859-e3ed28cd4f90", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-da109fcf", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:30:46Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-da109fcf"}, "namespace": "e2e-unlabeled-da109fcf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ed9d9587-e751-423e-9859-e3ed28cd4f90", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-unlabeled-da109fcf because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T21:30:46Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-da109fcf"}, "namespace": "e2e-unlabeled-da109fcf", "name": "odh-trusted-ca-bundle", "reconcileID": "20242ba0-e344-4c35-b5ee-db5f54d21a82", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-da109fcf"}
2026-07-10T21:30:46Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-da109fcf"}, "namespace": "e2e-unlabeled-da109fcf", "name": "odh-trusted-ca-bundle", "reconcileID": "20242ba0-e344-4c35-b5ee-db5f54d21a82", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-da109fcf"}
2026-07-10T21:30:46Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-da109fcf"}, "namespace": "e2e-unlabeled-da109fcf", "name": "openshift-service-ca.crt", "reconcileID": "84f7707e-daf0-4ae5-b56b-eee778ef03d0", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-da109fcf"}
2026-07-10T21:30:46Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-da109fcf"}, "namespace": "e2e-unlabeled-da109fcf", "name": "openshift-service-ca.crt", "reconcileID": "84f7707e-daf0-4ae5-b56b-eee778ef03d0", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-da109fcf"}
2026-07-10T21:30:46Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-da109fcf"}, "namespace": "e2e-unlabeled-da109fcf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d41b52f1-20b7-42a3-a41a-79cf876d1da7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-da109fcf"}
2026-07-10T21:30:46Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-da109fcf"}, "namespace": "e2e-unlabeled-da109fcf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d41b52f1-20b7-42a3-a41a-79cf876d1da7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-da109fcf"}
2026-07-10T21:30:46Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-b41bca67"}, "namespace": "ai-tenant-e2e-mt-b41bca67", "name": "openshift-service-ca.crt", "reconcileID": "1e374f93-53fc-4b71-8cd8-634cbaa04bfd", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-b41bca67"}
2026-07-10T21:30:46Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-b41bca67"}, "namespace": "ai-tenant-e2e-mt-b41bca67", "name": "odh-trusted-ca-bundle", "reconcileID": "799b7c7c-ce8a-487e-ac4c-b51cc4b1f57d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-b41bca67"}
2026-07-10T21:30:46Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-b41bca67"}, "namespace": "ai-tenant-e2e-mt-b41bca67", "name": "openshift-service-ca.crt", "reconcileID": "2b496861-f0a1-452f-a6b5-d1bc38888dda", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-b41bca67", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:30:46Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-b41bca67"}, "namespace": "ai-tenant-e2e-mt-b41bca67", "name": "openshift-service-ca.crt", "reconcileID": "2b496861-f0a1-452f-a6b5-d1bc38888dda", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-b41bca67"}
2026-07-10T21:30:46Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-b41bca67"}, "namespace": "ai-tenant-e2e-mt-b41bca67", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "323137c7-b42b-43a8-990f-88c2340ca146", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-b41bca67"}
2026-07-10T21:30:46Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-b41bca67"}, "namespace": "ai-tenant-e2e-mt-b41bca67", "name": "openshift-service-ca.crt", "reconcileID": "a9940d57-9df8-4929-b1a7-4146fd1ba483", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-b41bca67"}
2026-07-10T21:30:46Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-b41bca67"}, "namespace": "ai-tenant-e2e-mt-b41bca67", "name": "odh-trusted-ca-bundle", "reconcileID": "d5c2d345-f1d6-496a-889b-1951b577ce11", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-b41bca67", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:30:46Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-b41bca67"}, "namespace": "ai-tenant-e2e-mt-b41bca67", "name": "odh-trusted-ca-bundle", "reconcileID": "d5c2d345-f1d6-496a-889b-1951b577ce11", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-b41bca67"}
2026-07-10T21:30:46Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-b41bca67"}, "namespace": "ai-tenant-e2e-mt-b41bca67", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "98265889-047f-4da1-9c43-dfc598196309", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-b41bca67"}
2026-07-10T21:31:04Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-b41bca67"}, "namespace": "ai-tenant-e2e-mt-b41bca67", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "190bbc21-5c61-4cb3-8f38-e891883e215c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-b41bca67"}
2026-07-10T21:31:04Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-b41bca67"}, "namespace": "ai-tenant-e2e-mt-b41bca67", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "190bbc21-5c61-4cb3-8f38-e891883e215c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-b41bca67", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:31:04Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-b41bca67"}, "namespace": "ai-tenant-e2e-mt-b41bca67", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "190bbc21-5c61-4cb3-8f38-e891883e215c", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-b41bca67 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T21:31:04Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-b41bca67"}, "namespace": "ai-tenant-e2e-mt-b41bca67", "name": "odh-trusted-ca-bundle", "reconcileID": "24f1418a-6f17-4b49-a186-d18fb0b2c572", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-b41bca67"}
2026-07-10T21:31:04Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-b41bca67"}, "namespace": "ai-tenant-e2e-mt-b41bca67", "name": "odh-trusted-ca-bundle", "reconcileID": "24f1418a-6f17-4b49-a186-d18fb0b2c572", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-b41bca67"}
2026-07-10T21:31:04Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-b41bca67"}, "namespace": "ai-tenant-e2e-mt-b41bca67", "name": "openshift-service-ca.crt", "reconcileID": "79e8f525-d199-4ed2-a246-adbeaf131394", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-b41bca67"}
2026-07-10T21:31:04Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-b41bca67"}, "namespace": "ai-tenant-e2e-mt-b41bca67", "name": "openshift-service-ca.crt", "reconcileID": "79e8f525-d199-4ed2-a246-adbeaf131394", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-b41bca67"}
2026-07-10T21:31:04Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-b41bca67"}, "namespace": "ai-tenant-e2e-mt-b41bca67", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9ebb976e-cf18-4d8a-a933-6e51fe303c03", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-b41bca67"}
2026-07-10T21:31:04Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-b41bca67"}, "namespace": "ai-tenant-e2e-mt-b41bca67", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9ebb976e-cf18-4d8a-a933-6e51fe303c03", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-b41bca67"}
2026-07-10T21:31:04Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-66ae15ef"}, "namespace": "ai-tenant-e2e-mt-66ae15ef", "name": "openshift-service-ca.crt", "reconcileID": "1006c81a-128f-48d5-83c1-a049f3d6aa84", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-66ae15ef"}
2026-07-10T21:31:04Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-66ae15ef"}, "namespace": "ai-tenant-e2e-mt-66ae15ef", "name": "odh-trusted-ca-bundle", "reconcileID": "631eaacb-d865-4e9e-92d3-f28b78d829fa", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-66ae15ef"}
2026-07-10T21:31:04Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-66ae15ef"}, "namespace": "ai-tenant-e2e-mt-66ae15ef", "name": "openshift-service-ca.crt", "reconcileID": "8ed87fec-b61b-4577-921a-c7eae75040f9", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-66ae15ef", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:31:04Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-66ae15ef"}, "namespace": "ai-tenant-e2e-mt-66ae15ef", "name": "openshift-service-ca.crt", "reconcileID": "8ed87fec-b61b-4577-921a-c7eae75040f9", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-66ae15ef"}
2026-07-10T21:31:04Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-66ae15ef"}, "namespace": "ai-tenant-e2e-mt-66ae15ef", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "408bdd76-fb0d-4ff9-926b-6f8a47287057", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-66ae15ef"}
2026-07-10T21:31:04Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-66ae15ef"}, "namespace": "ai-tenant-e2e-mt-66ae15ef", "name": "openshift-service-ca.crt", "reconcileID": "0bccd8f9-eb34-46cb-a680-391ac26c2ed1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-66ae15ef"}
2026-07-10T21:31:04Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-66ae15ef"}, "namespace": "ai-tenant-e2e-mt-66ae15ef", "name": "odh-trusted-ca-bundle", "reconcileID": "304ca848-dd7b-4b39-87bc-09711065a7f2", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-66ae15ef", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:31:04Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-66ae15ef"}, "namespace": "ai-tenant-e2e-mt-66ae15ef", "name": "odh-trusted-ca-bundle", "reconcileID": "304ca848-dd7b-4b39-87bc-09711065a7f2", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-66ae15ef"}
2026-07-10T21:31:04Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-66ae15ef"}, "namespace": "ai-tenant-e2e-mt-66ae15ef", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2b7d6588-20a6-420f-8a98-69b021ca6f23", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-66ae15ef"}
2026-07-10T21:31:05Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "openshift-service-ca.crt", "reconcileID": "f083f2d1-1003-4030-9d7f-2218962a2ebe", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-24e02a52"}
2026-07-10T21:31:05Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "odh-trusted-ca-bundle", "reconcileID": "1eb14ace-e8c5-4600-bb7f-ff77623157da", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-24e02a52"}
2026-07-10T21:31:05Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "openshift-service-ca.crt", "reconcileID": "7abd63b4-ebdc-49c5-914e-63a9df493b65", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-24e02a52", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:31:05Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "openshift-service-ca.crt", "reconcileID": "7abd63b4-ebdc-49c5-914e-63a9df493b65", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-24e02a52"}
2026-07-10T21:31:05Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "openshift-service-ca.crt", "reconcileID": "b2081286-be76-4d78-942b-5b4d7e91d0ee", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-24e02a52", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:31:05Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "openshift-service-ca.crt", "reconcileID": "b2081286-be76-4d78-942b-5b4d7e91d0ee", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T21:31:05Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d83b90d7-44d4-43c2-8734-a834410ca751", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-24e02a52"}
2026-07-10T21:31:05Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "openshift-service-ca.crt", "reconcileID": "46ff678f-c0ae-4e6c-87cf-a2782585e71f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-24e02a52"}
2026-07-10T21:31:05Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "odh-trusted-ca-bundle", "reconcileID": "18e7322b-b67c-49a4-9063-2ce9986f9008", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-24e02a52", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:31:05Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "odh-trusted-ca-bundle", "reconcileID": "18e7322b-b67c-49a4-9063-2ce9986f9008", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-24e02a52"}
2026-07-10T21:31:05Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3276f560-6986-4748-a9ab-b3cc4aebe6ba", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-24e02a52"}
2026-07-10T21:31:12Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-66ae15ef"}, "namespace": "ai-tenant-e2e-mt-66ae15ef", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5e0b895b-8ba5-41c9-afa0-444999117c23", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-66ae15ef"}
2026-07-10T21:31:12Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-66ae15ef"}, "namespace": "ai-tenant-e2e-mt-66ae15ef", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5e0b895b-8ba5-41c9-afa0-444999117c23", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-66ae15ef", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:31:12Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-66ae15ef"}, "namespace": "ai-tenant-e2e-mt-66ae15ef", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5e0b895b-8ba5-41c9-afa0-444999117c23", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-66ae15ef because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T21:31:12Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-66ae15ef"}, "namespace": "ai-tenant-e2e-mt-66ae15ef", "name": "odh-trusted-ca-bundle", "reconcileID": "aa8738c2-09fe-4952-9081-bad7877ca83e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-66ae15ef"}
2026-07-10T21:31:12Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-66ae15ef"}, "namespace": "ai-tenant-e2e-mt-66ae15ef", "name": "odh-trusted-ca-bundle", "reconcileID": "aa8738c2-09fe-4952-9081-bad7877ca83e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-66ae15ef"}
2026-07-10T21:31:12Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-66ae15ef"}, "namespace": "ai-tenant-e2e-mt-66ae15ef", "name": "openshift-service-ca.crt", "reconcileID": "64b7e97a-c2a3-40e8-bf0c-4b593572cbe0", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-66ae15ef"}
2026-07-10T21:31:12Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-66ae15ef"}, "namespace": "ai-tenant-e2e-mt-66ae15ef", "name": "openshift-service-ca.crt", "reconcileID": "64b7e97a-c2a3-40e8-bf0c-4b593572cbe0", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-66ae15ef"}
2026-07-10T21:31:12Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-66ae15ef"}, "namespace": "ai-tenant-e2e-mt-66ae15ef", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a42e3069-e500-45ad-9588-455bce0a2a30", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-66ae15ef"}
2026-07-10T21:31:12Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-66ae15ef"}, "namespace": "ai-tenant-e2e-mt-66ae15ef", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a42e3069-e500-45ad-9588-455bce0a2a30", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-66ae15ef"}
2026-07-10T21:31:19Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a39e8184-2fe1-45cf-a500-2f0798834868", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-24e02a52"}
2026-07-10T21:31:19Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a39e8184-2fe1-45cf-a500-2f0798834868", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-24e02a52", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:31:19Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a39e8184-2fe1-45cf-a500-2f0798834868", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-24e02a52 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T21:31:19Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "77973930-bb58-4040-be46-ae69aade32db", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-24e02a52"}
2026-07-10T21:31:19Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "77973930-bb58-4040-be46-ae69aade32db", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-24e02a52", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:31:19Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "77973930-bb58-4040-be46-ae69aade32db", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-24e02a52 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T21:31:19Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "odh-trusted-ca-bundle", "reconcileID": "fe9d646d-339b-44ac-875a-0f0496e8764f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-24e02a52"}
2026-07-10T21:31:19Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "odh-trusted-ca-bundle", "reconcileID": "fe9d646d-339b-44ac-875a-0f0496e8764f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-24e02a52", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:31:19Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "odh-trusted-ca-bundle", "reconcileID": "fe9d646d-339b-44ac-875a-0f0496e8764f", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-24e02a52 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T21:31:19Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "openshift-service-ca.crt", "reconcileID": "dbac4a73-43d7-425e-b243-1ccde04a82cd", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-24e02a52"}
2026-07-10T21:31:19Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "openshift-service-ca.crt", "reconcileID": "dbac4a73-43d7-425e-b243-1ccde04a82cd", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-24e02a52"}
2026-07-10T21:31:19Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e15bb093-2282-45e5-bee9-e11f737d44c1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-24e02a52"}
2026-07-10T21:31:19Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e15bb093-2282-45e5-bee9-e11f737d44c1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-24e02a52"}
2026-07-10T21:31:19Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "odh-trusted-ca-bundle", "reconcileID": "e4e723f8-9d75-4c82-acc5-5cfa835e84bd", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-24e02a52"}
2026-07-10T21:31:19Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-24e02a52"}, "namespace": "ai-tenant-e2e-mt-24e02a52", "name": "odh-trusted-ca-bundle", "reconcileID": "e4e723f8-9d75-4c82-acc5-5cfa835e84bd", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-24e02a52"}
2026-07-10T21:31:20Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-59a51ffd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd", "reconcileID": "e0037b29-8e0e-48dd-90e8-3b79bd47b58c", "Gateway": "e2e-mt-59a51ffd", "namespace": "openshift-ingress"}
2026-07-10T21:31:20Z	INFO	Creating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-59a51ffd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd", "reconcileID": "e0037b29-8e0e-48dd-90e8-3b79bd47b58c", "Gateway": "e2e-mt-59a51ffd", "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd-authn-ssl"}
2026-07-10T21:31:20Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-59a51ffd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd", "reconcileID": "e0037b29-8e0e-48dd-90e8-3b79bd47b58c", "Gateway": "e2e-mt-59a51ffd", "namespace": "openshift-ingress"}
2026-07-10T21:31:20Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-59a51ffd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd", "reconcileID": "c0759062-094d-4386-bb7b-5468b750b7d9", "Gateway": "e2e-mt-59a51ffd", "namespace": "openshift-ingress"}
2026-07-10T21:31:20Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-59a51ffd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd", "reconcileID": "c0759062-094d-4386-bb7b-5468b750b7d9", "Gateway": "e2e-mt-59a51ffd", "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd-authn-ssl"}
2026-07-10T21:31:20Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-59a51ffd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd", "reconcileID": "c0759062-094d-4386-bb7b-5468b750b7d9", "Gateway": "e2e-mt-59a51ffd", "namespace": "openshift-ingress"}
2026-07-10T21:31:20Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-59a51ffd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd", "reconcileID": "4264c78b-5a2d-4a48-86ea-eb5887b1590e", "Gateway": "e2e-mt-59a51ffd", "namespace": "openshift-ingress"}
2026-07-10T21:31:20Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-59a51ffd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd", "reconcileID": "4264c78b-5a2d-4a48-86ea-eb5887b1590e", "Gateway": "e2e-mt-59a51ffd", "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd-authn-ssl"}
2026-07-10T21:31:20Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-59a51ffd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd", "reconcileID": "4264c78b-5a2d-4a48-86ea-eb5887b1590e", "Gateway": "e2e-mt-59a51ffd", "namespace": "openshift-ingress"}
2026-07-10T21:31:20Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-59a51ffd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd", "reconcileID": "a6473617-b085-4249-83fe-f6a6ead85396", "Gateway": "e2e-mt-59a51ffd", "namespace": "openshift-ingress"}
2026-07-10T21:31:20Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-59a51ffd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd", "reconcileID": "a6473617-b085-4249-83fe-f6a6ead85396", "Gateway": "e2e-mt-59a51ffd", "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd-authn-ssl"}
2026-07-10T21:31:20Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-59a51ffd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd", "reconcileID": "a6473617-b085-4249-83fe-f6a6ead85396", "Gateway": "e2e-mt-59a51ffd", "namespace": "openshift-ingress"}
2026-07-10T21:31:20Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-59a51ffd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd", "reconcileID": "c904174b-5e8c-4754-83c5-2d5e97542e52", "Gateway": "e2e-mt-59a51ffd", "namespace": "openshift-ingress"}
2026-07-10T21:31:20Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-59a51ffd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd", "reconcileID": "c904174b-5e8c-4754-83c5-2d5e97542e52", "Gateway": "e2e-mt-59a51ffd", "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd-authn-ssl"}
2026-07-10T21:31:20Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-59a51ffd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd", "reconcileID": "c904174b-5e8c-4754-83c5-2d5e97542e52", "Gateway": "e2e-mt-59a51ffd", "namespace": "openshift-ingress"}
2026-07-10T21:31:20Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-59a51ffd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd", "reconcileID": "2c42b5b9-6fb1-4a3c-a800-45b3a3d2c0da", "Gateway": "e2e-mt-59a51ffd", "namespace": "openshift-ingress"}
2026-07-10T21:31:20Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-59a51ffd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd", "reconcileID": "2c42b5b9-6fb1-4a3c-a800-45b3a3d2c0da", "Gateway": "e2e-mt-59a51ffd", "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd-authn-ssl"}
2026-07-10T21:31:20Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-59a51ffd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd", "reconcileID": "2c42b5b9-6fb1-4a3c-a800-45b3a3d2c0da", "Gateway": "e2e-mt-59a51ffd", "namespace": "openshift-ingress"}
2026-07-10T21:31:21Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-59a51ffd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd", "reconcileID": "9d7bc188-a559-4818-8936-516e3827f0c9", "Gateway": "e2e-mt-59a51ffd", "namespace": "openshift-ingress"}
2026-07-10T21:31:21Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-59a51ffd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd", "reconcileID": "9d7bc188-a559-4818-8936-516e3827f0c9", "Gateway": "e2e-mt-59a51ffd", "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd-authn-ssl"}
2026-07-10T21:31:21Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-59a51ffd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd", "reconcileID": "9d7bc188-a559-4818-8936-516e3827f0c9", "Gateway": "e2e-mt-59a51ffd", "namespace": "openshift-ingress"}
2026-07-10T21:31:23Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-59a51ffd"}, "namespace": "ai-tenant-e2e-mt-59a51ffd", "name": "openshift-service-ca.crt", "reconcileID": "ff149618-0a44-4546-8c93-292f55f4369f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-59a51ffd"}
2026-07-10T21:31:23Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-59a51ffd"}, "namespace": "ai-tenant-e2e-mt-59a51ffd", "name": "odh-trusted-ca-bundle", "reconcileID": "2879775a-d87a-4d60-89f2-ea017ee5c580", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-59a51ffd"}
2026-07-10T21:31:23Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-59a51ffd"}, "namespace": "ai-tenant-e2e-mt-59a51ffd", "name": "openshift-service-ca.crt", "reconcileID": "9bc92142-bdea-4db7-ac0f-62ba1c96d988", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-59a51ffd", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:31:23Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-59a51ffd"}, "namespace": "ai-tenant-e2e-mt-59a51ffd", "name": "openshift-service-ca.crt", "reconcileID": "9bc92142-bdea-4db7-ac0f-62ba1c96d988", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-59a51ffd"}
2026-07-10T21:31:23Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-59a51ffd"}, "namespace": "ai-tenant-e2e-mt-59a51ffd", "name": "openshift-service-ca.crt", "reconcileID": "586b02db-27a3-4280-8981-6dfe5ae11501", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-59a51ffd"}
2026-07-10T21:31:23Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-59a51ffd"}, "namespace": "ai-tenant-e2e-mt-59a51ffd", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "30c728b6-226e-4962-9c16-ac83c85a89e3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-59a51ffd"}
2026-07-10T21:31:23Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-59a51ffd"}, "namespace": "ai-tenant-e2e-mt-59a51ffd", "name": "odh-trusted-ca-bundle", "reconcileID": "dafe791f-c931-40db-983a-750a1250c5d2", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-59a51ffd", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:31:23Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-59a51ffd"}, "namespace": "ai-tenant-e2e-mt-59a51ffd", "name": "odh-trusted-ca-bundle", "reconcileID": "dafe791f-c931-40db-983a-750a1250c5d2", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-59a51ffd"}
2026-07-10T21:31:23Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-59a51ffd"}, "namespace": "ai-tenant-e2e-mt-59a51ffd", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a08e2bbf-7df7-45c1-8d73-c770eee06418", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-59a51ffd"}
2026-07-10T21:34:38Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-59a51ffd"}, "namespace": "ai-tenant-e2e-mt-59a51ffd", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ddb9eb5f-91de-4794-b516-d8635fe930c7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-59a51ffd"}
2026-07-10T21:34:38Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-59a51ffd"}, "namespace": "ai-tenant-e2e-mt-59a51ffd", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ddb9eb5f-91de-4794-b516-d8635fe930c7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-59a51ffd", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:34:38Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-59a51ffd"}, "namespace": "ai-tenant-e2e-mt-59a51ffd", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ddb9eb5f-91de-4794-b516-d8635fe930c7", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-59a51ffd because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T21:34:38Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-59a51ffd"}, "namespace": "ai-tenant-e2e-mt-59a51ffd", "name": "odh-trusted-ca-bundle", "reconcileID": "9f1a197b-48c9-4ed1-8739-dff1506a6fcf", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-59a51ffd"}
2026-07-10T21:34:38Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-59a51ffd"}, "namespace": "ai-tenant-e2e-mt-59a51ffd", "name": "odh-trusted-ca-bundle", "reconcileID": "9f1a197b-48c9-4ed1-8739-dff1506a6fcf", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-59a51ffd"}
2026-07-10T21:34:38Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-59a51ffd"}, "namespace": "ai-tenant-e2e-mt-59a51ffd", "name": "openshift-service-ca.crt", "reconcileID": "587f7c45-8bd6-47a0-8bb6-5317f29c4d4c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-59a51ffd"}
2026-07-10T21:34:38Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-59a51ffd"}, "namespace": "ai-tenant-e2e-mt-59a51ffd", "name": "openshift-service-ca.crt", "reconcileID": "587f7c45-8bd6-47a0-8bb6-5317f29c4d4c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-59a51ffd"}
2026-07-10T21:34:38Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-59a51ffd"}, "namespace": "ai-tenant-e2e-mt-59a51ffd", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9553feeb-0a31-43eb-bc52-7bc0572bc37a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-59a51ffd"}
2026-07-10T21:34:38Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-59a51ffd"}, "namespace": "ai-tenant-e2e-mt-59a51ffd", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9553feeb-0a31-43eb-bc52-7bc0572bc37a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-59a51ffd"}
2026-07-10T21:34:39Z	DEBUG	Gateway not found, resources owned by Gateway will be garbage collected	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-59a51ffd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-59a51ffd", "reconcileID": "f4a3c3ac-bb6a-4824-bf8e-3399c4a9dd2f", "Gateway": "e2e-mt-59a51ffd", "namespace": "openshift-ingress"}
2026-07-10T21:34:39Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-a21c5305"}, "namespace": "e2e-webhook-sub-a21c5305", "name": "openshift-service-ca.crt", "reconcileID": "a0d869d7-452c-4b94-bd29-8d2bb2a9e563", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-a21c5305"}
2026-07-10T21:34:39Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-a21c5305"}, "namespace": "e2e-webhook-sub-a21c5305", "name": "odh-trusted-ca-bundle", "reconcileID": "300e91d5-a903-497c-a103-dc41747bc0a7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-a21c5305"}
2026-07-10T21:34:39Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-a21c5305"}, "namespace": "e2e-webhook-sub-a21c5305", "name": "openshift-service-ca.crt", "reconcileID": "ff08a643-14fe-4a95-aa54-4834db90081f", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-a21c5305", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:34:39Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-a21c5305"}, "namespace": "e2e-webhook-sub-a21c5305", "name": "openshift-service-ca.crt", "reconcileID": "ff08a643-14fe-4a95-aa54-4834db90081f", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-a21c5305"}
2026-07-10T21:34:39Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-a21c5305"}, "namespace": "e2e-webhook-sub-a21c5305", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "69a592af-2270-4e88-b785-9e5bd72c4843", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-a21c5305"}
2026-07-10T21:34:39Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-a21c5305"}, "namespace": "e2e-webhook-sub-a21c5305", "name": "openshift-service-ca.crt", "reconcileID": "32094fba-2423-4d80-b975-6cb7351507d5", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-a21c5305"}
2026-07-10T21:34:39Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-a21c5305"}, "namespace": "e2e-webhook-sub-a21c5305", "name": "odh-trusted-ca-bundle", "reconcileID": "e4282c1f-4a40-4d25-b4cd-6249af84a0ff", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-a21c5305", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:34:39Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-a21c5305"}, "namespace": "e2e-webhook-sub-a21c5305", "name": "odh-trusted-ca-bundle", "reconcileID": "e4282c1f-4a40-4d25-b4cd-6249af84a0ff", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-a21c5305"}
2026-07-10T21:34:39Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-a21c5305"}, "namespace": "e2e-webhook-sub-a21c5305", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ca1914d6-4f70-4c4e-9024-fa5683b2141e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-a21c5305"}
2026-07-10T21:34:45Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-a21c5305"}, "namespace": "e2e-webhook-sub-a21c5305", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6e392fb7-df87-47ce-ad24-ac35e660f422", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-a21c5305"}
2026-07-10T21:34:45Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-a21c5305"}, "namespace": "e2e-webhook-sub-a21c5305", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6e392fb7-df87-47ce-ad24-ac35e660f422", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-a21c5305", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:34:45Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-a21c5305"}, "namespace": "e2e-webhook-sub-a21c5305", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6e392fb7-df87-47ce-ad24-ac35e660f422", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-sub-a21c5305 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T21:34:45Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-a21c5305"}, "namespace": "e2e-webhook-sub-a21c5305", "name": "odh-trusted-ca-bundle", "reconcileID": "28d04efe-ee39-4e95-8a11-e2bf56aa0724", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-a21c5305"}
2026-07-10T21:34:45Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-a21c5305"}, "namespace": "e2e-webhook-sub-a21c5305", "name": "odh-trusted-ca-bundle", "reconcileID": "28d04efe-ee39-4e95-8a11-e2bf56aa0724", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-a21c5305"}
2026-07-10T21:34:45Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-a21c5305"}, "namespace": "e2e-webhook-sub-a21c5305", "name": "openshift-service-ca.crt", "reconcileID": "34c589f9-8eec-4807-ae59-05b703dde34e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-a21c5305"}
2026-07-10T21:34:45Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-a21c5305"}, "namespace": "e2e-webhook-sub-a21c5305", "name": "openshift-service-ca.crt", "reconcileID": "34c589f9-8eec-4807-ae59-05b703dde34e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-a21c5305"}
2026-07-10T21:34:45Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-a21c5305"}, "namespace": "e2e-webhook-sub-a21c5305", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0a6b5ef6-3c4f-4aad-808d-9326c59c3554", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-a21c5305"}
2026-07-10T21:34:45Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-a21c5305"}, "namespace": "e2e-webhook-sub-a21c5305", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0a6b5ef6-3c4f-4aad-808d-9326c59c3554", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-a21c5305"}
2026-07-10T21:34:45Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-c79d975d"}, "namespace": "e2e-webhook-auth-c79d975d", "name": "openshift-service-ca.crt", "reconcileID": "02c8ebd6-7625-4d46-89e7-0838015e9c1e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-c79d975d"}
2026-07-10T21:34:45Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-c79d975d"}, "namespace": "e2e-webhook-auth-c79d975d", "name": "odh-trusted-ca-bundle", "reconcileID": "e6688e83-af88-449c-b028-7660aa2fafc4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-c79d975d"}
2026-07-10T21:34:45Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-c79d975d"}, "namespace": "e2e-webhook-auth-c79d975d", "name": "openshift-service-ca.crt", "reconcileID": "bd6bde85-37af-4a66-82be-a04d1faecfad", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-c79d975d", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:34:45Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-c79d975d"}, "namespace": "e2e-webhook-auth-c79d975d", "name": "openshift-service-ca.crt", "reconcileID": "bd6bde85-37af-4a66-82be-a04d1faecfad", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-c79d975d"}
2026-07-10T21:34:45Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-c79d975d"}, "namespace": "e2e-webhook-auth-c79d975d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b348b27e-60ed-4770-ba5b-d9fdbc0e5110", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-c79d975d"}
2026-07-10T21:34:45Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-c79d975d"}, "namespace": "e2e-webhook-auth-c79d975d", "name": "openshift-service-ca.crt", "reconcileID": "f255fab1-3420-4f52-9749-0cafbdd9fea8", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-c79d975d"}
2026-07-10T21:34:45Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-c79d975d"}, "namespace": "e2e-webhook-auth-c79d975d", "name": "odh-trusted-ca-bundle", "reconcileID": "d2d6e42d-8e6b-48dc-8a45-9fa94ca23263", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-c79d975d", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:34:45Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-c79d975d"}, "namespace": "e2e-webhook-auth-c79d975d", "name": "odh-trusted-ca-bundle", "reconcileID": "d2d6e42d-8e6b-48dc-8a45-9fa94ca23263", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-c79d975d"}
2026-07-10T21:34:45Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-c79d975d"}, "namespace": "e2e-webhook-auth-c79d975d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f9d8e59e-759d-429f-9b81-c7a41692fd30", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-c79d975d"}
2026-07-10T21:34:51Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-c79d975d"}, "namespace": "e2e-webhook-auth-c79d975d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5b1857c9-37b5-4c6e-9097-d8fd9a4a5d44", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-c79d975d"}
2026-07-10T21:34:51Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-c79d975d"}, "namespace": "e2e-webhook-auth-c79d975d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5b1857c9-37b5-4c6e-9097-d8fd9a4a5d44", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-c79d975d", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:34:51Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-c79d975d"}, "namespace": "e2e-webhook-auth-c79d975d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5b1857c9-37b5-4c6e-9097-d8fd9a4a5d44", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-auth-c79d975d because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T21:34:51Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-c79d975d"}, "namespace": "e2e-webhook-auth-c79d975d", "name": "odh-trusted-ca-bundle", "reconcileID": "fa584731-63a1-4d8c-b890-33bac0587569", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-c79d975d"}
2026-07-10T21:34:51Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-c79d975d"}, "namespace": "e2e-webhook-auth-c79d975d", "name": "odh-trusted-ca-bundle", "reconcileID": "fa584731-63a1-4d8c-b890-33bac0587569", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-c79d975d"}
2026-07-10T21:34:51Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-c79d975d"}, "namespace": "e2e-webhook-auth-c79d975d", "name": "openshift-service-ca.crt", "reconcileID": "a158e4ec-a336-4ee3-8869-3c514b28bdd5", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-c79d975d"}
2026-07-10T21:34:51Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-c79d975d"}, "namespace": "e2e-webhook-auth-c79d975d", "name": "openshift-service-ca.crt", "reconcileID": "a158e4ec-a336-4ee3-8869-3c514b28bdd5", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-c79d975d"}
2026-07-10T21:34:51Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-c79d975d"}, "namespace": "e2e-webhook-auth-c79d975d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f257b58f-246b-4b60-8f24-3ca57668d28c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-c79d975d"}
2026-07-10T21:34:51Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-c79d975d"}, "namespace": "e2e-webhook-auth-c79d975d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f257b58f-246b-4b60-8f24-3ca57668d28c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-c79d975d"}
2026-07-10T21:34:58Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-9dc46fc9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9", "reconcileID": "636fe82c-ba0b-4db9-81e5-7ce8f535e4b8", "Gateway": "e2e-mt-9dc46fc9", "namespace": "openshift-ingress"}
2026-07-10T21:34:58Z	INFO	Creating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-9dc46fc9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9", "reconcileID": "636fe82c-ba0b-4db9-81e5-7ce8f535e4b8", "Gateway": "e2e-mt-9dc46fc9", "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9-authn-ssl"}
2026-07-10T21:34:58Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-9dc46fc9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9", "reconcileID": "636fe82c-ba0b-4db9-81e5-7ce8f535e4b8", "Gateway": "e2e-mt-9dc46fc9", "namespace": "openshift-ingress"}
2026-07-10T21:34:58Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-9dc46fc9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9", "reconcileID": "cd9d5463-f44d-44e1-943a-d5c8aaa20f79", "Gateway": "e2e-mt-9dc46fc9", "namespace": "openshift-ingress"}
2026-07-10T21:34:58Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-9dc46fc9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9", "reconcileID": "cd9d5463-f44d-44e1-943a-d5c8aaa20f79", "Gateway": "e2e-mt-9dc46fc9", "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9-authn-ssl"}
2026-07-10T21:34:58Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-9dc46fc9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9", "reconcileID": "cd9d5463-f44d-44e1-943a-d5c8aaa20f79", "Gateway": "e2e-mt-9dc46fc9", "namespace": "openshift-ingress"}
2026-07-10T21:34:58Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-9dc46fc9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9", "reconcileID": "0d11d9b8-3e5a-4211-9af2-094d3d8ab25e", "Gateway": "e2e-mt-9dc46fc9", "namespace": "openshift-ingress"}
2026-07-10T21:34:58Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-9dc46fc9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9", "reconcileID": "0d11d9b8-3e5a-4211-9af2-094d3d8ab25e", "Gateway": "e2e-mt-9dc46fc9", "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9-authn-ssl"}
2026-07-10T21:34:58Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-9dc46fc9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9", "reconcileID": "0d11d9b8-3e5a-4211-9af2-094d3d8ab25e", "Gateway": "e2e-mt-9dc46fc9", "namespace": "openshift-ingress"}
2026-07-10T21:34:58Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-9dc46fc9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9", "reconcileID": "43b1ba63-c712-4f64-a33f-f892ba1efadb", "Gateway": "e2e-mt-9dc46fc9", "namespace": "openshift-ingress"}
2026-07-10T21:34:58Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-9dc46fc9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9", "reconcileID": "43b1ba63-c712-4f64-a33f-f892ba1efadb", "Gateway": "e2e-mt-9dc46fc9", "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9-authn-ssl"}
2026-07-10T21:34:58Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-9dc46fc9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9", "reconcileID": "43b1ba63-c712-4f64-a33f-f892ba1efadb", "Gateway": "e2e-mt-9dc46fc9", "namespace": "openshift-ingress"}
2026-07-10T21:34:58Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-9dc46fc9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9", "reconcileID": "8108416f-9e6d-47fc-9fdd-f9c41ca5b1a3", "Gateway": "e2e-mt-9dc46fc9", "namespace": "openshift-ingress"}
2026-07-10T21:34:58Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-9dc46fc9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9", "reconcileID": "8108416f-9e6d-47fc-9fdd-f9c41ca5b1a3", "Gateway": "e2e-mt-9dc46fc9", "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9-authn-ssl"}
2026-07-10T21:34:58Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-9dc46fc9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9", "reconcileID": "8108416f-9e6d-47fc-9fdd-f9c41ca5b1a3", "Gateway": "e2e-mt-9dc46fc9", "namespace": "openshift-ingress"}
2026-07-10T21:34:59Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-9dc46fc9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9", "reconcileID": "451c0c41-ef4d-411e-977a-ff22184fd9fd", "Gateway": "e2e-mt-9dc46fc9", "namespace": "openshift-ingress"}
2026-07-10T21:34:59Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-9dc46fc9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9", "reconcileID": "451c0c41-ef4d-411e-977a-ff22184fd9fd", "Gateway": "e2e-mt-9dc46fc9", "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9-authn-ssl"}
2026-07-10T21:34:59Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-9dc46fc9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9", "reconcileID": "451c0c41-ef4d-411e-977a-ff22184fd9fd", "Gateway": "e2e-mt-9dc46fc9", "namespace": "openshift-ingress"}
2026-07-10T21:35:00Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-9dc46fc9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9", "reconcileID": "1cdb2a7c-9dde-4ba2-aa8e-1a2af7ebc812", "Gateway": "e2e-mt-9dc46fc9", "namespace": "openshift-ingress"}
2026-07-10T21:35:00Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-9dc46fc9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9", "reconcileID": "1cdb2a7c-9dde-4ba2-aa8e-1a2af7ebc812", "Gateway": "e2e-mt-9dc46fc9", "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9-authn-ssl"}
2026-07-10T21:35:00Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-9dc46fc9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9", "reconcileID": "1cdb2a7c-9dde-4ba2-aa8e-1a2af7ebc812", "Gateway": "e2e-mt-9dc46fc9", "namespace": "openshift-ingress"}
2026-07-10T21:35:02Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9dc46fc9"}, "namespace": "ai-tenant-e2e-mt-9dc46fc9", "name": "openshift-service-ca.crt", "reconcileID": "b90bb45e-d61d-43e2-8a99-018fa2e448ee", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9dc46fc9"}
2026-07-10T21:35:02Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9dc46fc9"}, "namespace": "ai-tenant-e2e-mt-9dc46fc9", "name": "openshift-service-ca.crt", "reconcileID": "19226df5-0c43-4be4-990e-f862a29edf63", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9dc46fc9", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:35:02Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9dc46fc9"}, "namespace": "ai-tenant-e2e-mt-9dc46fc9", "name": "openshift-service-ca.crt", "reconcileID": "19226df5-0c43-4be4-990e-f862a29edf63", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9dc46fc9"}
2026-07-10T21:35:02Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-9dc46fc9"}, "namespace": "ai-tenant-e2e-mt-9dc46fc9", "name": "odh-trusted-ca-bundle", "reconcileID": "940e2da7-139c-4f9a-ba6e-73cd81f64f56", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-9dc46fc9", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:35:02Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-9dc46fc9"}, "namespace": "ai-tenant-e2e-mt-9dc46fc9", "name": "odh-trusted-ca-bundle", "reconcileID": "940e2da7-139c-4f9a-ba6e-73cd81f64f56", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T21:35:02Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9dc46fc9"}, "namespace": "ai-tenant-e2e-mt-9dc46fc9", "name": "openshift-service-ca.crt", "reconcileID": "ad60ccb7-b653-4e82-a3e6-bc7f24397b42", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9dc46fc9"}
2026-07-10T21:35:02Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9dc46fc9"}, "namespace": "ai-tenant-e2e-mt-9dc46fc9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a70e4a3f-4dd9-44fa-9c4b-cb3a0904820a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9dc46fc9"}
2026-07-10T21:35:02Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-9dc46fc9"}, "namespace": "ai-tenant-e2e-mt-9dc46fc9", "name": "odh-trusted-ca-bundle", "reconcileID": "5d8d626b-f6f1-42cd-b0a9-e282ee8e1159", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-9dc46fc9"}
2026-07-10T21:35:02Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-9dc46fc9"}, "namespace": "ai-tenant-e2e-mt-9dc46fc9", "name": "odh-trusted-ca-bundle", "reconcileID": "47648cfa-64e3-4607-8c62-ddbe1a0cf22d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-9dc46fc9", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:35:02Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-9dc46fc9"}, "namespace": "ai-tenant-e2e-mt-9dc46fc9", "name": "odh-trusted-ca-bundle", "reconcileID": "47648cfa-64e3-4607-8c62-ddbe1a0cf22d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-9dc46fc9"}
2026-07-10T21:35:02Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9dc46fc9"}, "namespace": "ai-tenant-e2e-mt-9dc46fc9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1882357a-2bb4-4641-ac5d-93582ccd9e9b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9dc46fc9"}
2026-07-10T21:37:50Z	INFO	No monitoring namespace detected, skipping monitoring reconciliation.	{"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"}
2026-07-10T21:37:50Z	INFO	Multi Node reconciling.	{"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"}
2026-07-10T21:37:52Z	INFO	ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt	{"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"}
2026-07-10T21:37:52Z	INFO	Multi Node reconciled successfully.	{"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"}
2026-07-10T21:38:17Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9dc46fc9"}, "namespace": "ai-tenant-e2e-mt-9dc46fc9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e9ca8538-d328-416b-943a-ff75d69811e3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9dc46fc9"}
2026-07-10T21:38:17Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9dc46fc9"}, "namespace": "ai-tenant-e2e-mt-9dc46fc9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e9ca8538-d328-416b-943a-ff75d69811e3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9dc46fc9", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:38:17Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9dc46fc9"}, "namespace": "ai-tenant-e2e-mt-9dc46fc9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e9ca8538-d328-416b-943a-ff75d69811e3", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-9dc46fc9 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T21:38:17Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-9dc46fc9"}, "namespace": "ai-tenant-e2e-mt-9dc46fc9", "name": "odh-trusted-ca-bundle", "reconcileID": "053789d2-de9e-45fd-b903-525ef0a3790b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-9dc46fc9"}
2026-07-10T21:38:17Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-9dc46fc9"}, "namespace": "ai-tenant-e2e-mt-9dc46fc9", "name": "odh-trusted-ca-bundle", "reconcileID": "053789d2-de9e-45fd-b903-525ef0a3790b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-9dc46fc9"}
2026-07-10T21:38:17Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9dc46fc9"}, "namespace": "ai-tenant-e2e-mt-9dc46fc9", "name": "openshift-service-ca.crt", "reconcileID": "ff019dae-caa9-453b-ad80-e0412f4bca05", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9dc46fc9"}
2026-07-10T21:38:17Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9dc46fc9"}, "namespace": "ai-tenant-e2e-mt-9dc46fc9", "name": "openshift-service-ca.crt", "reconcileID": "ff019dae-caa9-453b-ad80-e0412f4bca05", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9dc46fc9"}
2026-07-10T21:38:17Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9dc46fc9"}, "namespace": "ai-tenant-e2e-mt-9dc46fc9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a0de9741-e970-4797-8ab2-7a4294d1d663", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9dc46fc9"}
2026-07-10T21:38:17Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9dc46fc9"}, "namespace": "ai-tenant-e2e-mt-9dc46fc9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a0de9741-e970-4797-8ab2-7a4294d1d663", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9dc46fc9"}
2026-07-10T21:38:17Z	DEBUG	Gateway not found, resources owned by Gateway will be garbage collected	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-9dc46fc9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-9dc46fc9", "reconcileID": "19a0b6e7-e956-40c0-903d-684f52a83f49", "Gateway": "e2e-mt-9dc46fc9", "namespace": "openshift-ingress"}
2026-07-10T21:38:19Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-7fbe928a"}, "namespace": "ai-tenant-e2e-mt-7fbe928a", "name": "openshift-service-ca.crt", "reconcileID": "dc241185-42a7-46cf-bcd2-73d4672b63c9", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-7fbe928a"}
2026-07-10T21:38:19Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-7fbe928a"}, "namespace": "ai-tenant-e2e-mt-7fbe928a", "name": "odh-trusted-ca-bundle", "reconcileID": "4a3a1a84-5502-4409-8437-2a5fd70b21f7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-7fbe928a", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:38:19Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-7fbe928a"}, "namespace": "ai-tenant-e2e-mt-7fbe928a", "name": "odh-trusted-ca-bundle", "reconcileID": "4a3a1a84-5502-4409-8437-2a5fd70b21f7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-7fbe928a"}
2026-07-10T21:38:19Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-7fbe928a"}, "namespace": "ai-tenant-e2e-mt-7fbe928a", "name": "openshift-service-ca.crt", "reconcileID": "d3438693-02f8-4a20-af36-c2dc3caed822", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-7fbe928a"}
2026-07-10T21:38:19Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-7fbe928a"}, "namespace": "ai-tenant-e2e-mt-7fbe928a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "06006bf9-8cb6-45ff-88c2-25ceff801075", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-7fbe928a"}
2026-07-10T21:38:19Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-7fbe928a"}, "namespace": "ai-tenant-e2e-mt-7fbe928a", "name": "odh-trusted-ca-bundle", "reconcileID": "a625f729-6629-4fe2-a97a-09cc363a7eaf", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-7fbe928a", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:38:19Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-7fbe928a"}, "namespace": "ai-tenant-e2e-mt-7fbe928a", "name": "odh-trusted-ca-bundle", "reconcileID": "a625f729-6629-4fe2-a97a-09cc363a7eaf", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-7fbe928a"}
2026-07-10T21:38:19Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-7fbe928a"}, "namespace": "ai-tenant-e2e-mt-7fbe928a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "454d95ee-f2c9-41f0-9995-fc2a59ef64f1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-7fbe928a"}
2026-07-10T21:38:20Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-92cec1b2"}, "namespace": "ai-tenant-e2e-mt-92cec1b2", "name": "openshift-service-ca.crt", "reconcileID": "7607af56-6b23-43e2-a588-bbe5b1bbeb9e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-92cec1b2"}
2026-07-10T21:38:20Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-92cec1b2"}, "namespace": "ai-tenant-e2e-mt-92cec1b2", "name": "odh-trusted-ca-bundle", "reconcileID": "ba90278f-bc57-4cb0-913e-652ce3995f36", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-92cec1b2"}
2026-07-10T21:38:20Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-92cec1b2"}, "namespace": "ai-tenant-e2e-mt-92cec1b2", "name": "openshift-service-ca.crt", "reconcileID": "4ba32b31-a39b-4311-9830-bb4e0760078e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-92cec1b2", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:38:20Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-92cec1b2"}, "namespace": "ai-tenant-e2e-mt-92cec1b2", "name": "openshift-service-ca.crt", "reconcileID": "4ba32b31-a39b-4311-9830-bb4e0760078e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-92cec1b2"}
2026-07-10T21:38:20Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-92cec1b2"}, "namespace": "ai-tenant-e2e-mt-92cec1b2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e7cc6f3d-5cd3-4155-9649-78f6947bbf97", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-92cec1b2"}
2026-07-10T21:38:20Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-92cec1b2"}, "namespace": "ai-tenant-e2e-mt-92cec1b2", "name": "openshift-service-ca.crt", "reconcileID": "1637614c-2984-4314-85f5-c4ac97726881", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-92cec1b2"}
2026-07-10T21:38:20Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-92cec1b2"}, "namespace": "ai-tenant-e2e-mt-92cec1b2", "name": "odh-trusted-ca-bundle", "reconcileID": "77fcc4fa-ba4c-4ee4-922e-d4507bf8d196", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-92cec1b2", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:38:20Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-92cec1b2"}, "namespace": "ai-tenant-e2e-mt-92cec1b2", "name": "odh-trusted-ca-bundle", "reconcileID": "77fcc4fa-ba4c-4ee4-922e-d4507bf8d196", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-92cec1b2"}
2026-07-10T21:38:20Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-92cec1b2"}, "namespace": "ai-tenant-e2e-mt-92cec1b2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "20f47522-5e96-4833-a501-199c2b3fe379", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-92cec1b2"}
2026-07-10T21:38:27Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-7fbe928a"}, "namespace": "ai-tenant-e2e-mt-7fbe928a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "02d1febd-cdb6-4c6b-a413-97256270fc43", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-7fbe928a"}
2026-07-10T21:38:27Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-7fbe928a"}, "namespace": "ai-tenant-e2e-mt-7fbe928a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "02d1febd-cdb6-4c6b-a413-97256270fc43", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-7fbe928a", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:38:27Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-7fbe928a"}, "namespace": "ai-tenant-e2e-mt-7fbe928a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "02d1febd-cdb6-4c6b-a413-97256270fc43", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-7fbe928a because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T21:38:27Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-7fbe928a"}, "namespace": "ai-tenant-e2e-mt-7fbe928a", "name": "odh-trusted-ca-bundle", "reconcileID": "eabc98b3-61e3-4578-b3f4-c44ecfd360a3", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-7fbe928a"}
2026-07-10T21:38:27Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-7fbe928a"}, "namespace": "ai-tenant-e2e-mt-7fbe928a", "name": "odh-trusted-ca-bundle", "reconcileID": "eabc98b3-61e3-4578-b3f4-c44ecfd360a3", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-7fbe928a"}
2026-07-10T21:38:27Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-7fbe928a"}, "namespace": "ai-tenant-e2e-mt-7fbe928a", "name": "openshift-service-ca.crt", "reconcileID": "4e135f32-b7e5-4eb2-a8b5-c7dc8ad1a881", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-7fbe928a"}
2026-07-10T21:38:27Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-7fbe928a"}, "namespace": "ai-tenant-e2e-mt-7fbe928a", "name": "openshift-service-ca.crt", "reconcileID": "4e135f32-b7e5-4eb2-a8b5-c7dc8ad1a881", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-7fbe928a"}
2026-07-10T21:38:27Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-7fbe928a"}, "namespace": "ai-tenant-e2e-mt-7fbe928a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5ecd6dc1-a03f-483b-b22e-6b2d82d6820c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-7fbe928a"}
2026-07-10T21:38:27Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-7fbe928a"}, "namespace": "ai-tenant-e2e-mt-7fbe928a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5ecd6dc1-a03f-483b-b22e-6b2d82d6820c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-7fbe928a"}
2026-07-10T21:38:35Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-92cec1b2"}, "namespace": "ai-tenant-e2e-mt-92cec1b2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1b6f0198-8c24-459d-b791-580f8af0d8d2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-92cec1b2"}
2026-07-10T21:38:35Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-92cec1b2"}, "namespace": "ai-tenant-e2e-mt-92cec1b2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1b6f0198-8c24-459d-b791-580f8af0d8d2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-92cec1b2", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:38:35Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-92cec1b2"}, "namespace": "ai-tenant-e2e-mt-92cec1b2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1b6f0198-8c24-459d-b791-580f8af0d8d2", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-92cec1b2 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T21:38:35Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-92cec1b2"}, "namespace": "ai-tenant-e2e-mt-92cec1b2", "name": "odh-trusted-ca-bundle", "reconcileID": "63a46673-0d94-455e-b97b-050739968eb9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-92cec1b2"}
2026-07-10T21:38:35Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-92cec1b2"}, "namespace": "ai-tenant-e2e-mt-92cec1b2", "name": "odh-trusted-ca-bundle", "reconcileID": "63a46673-0d94-455e-b97b-050739968eb9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-92cec1b2"}
2026-07-10T21:38:35Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-92cec1b2"}, "namespace": "ai-tenant-e2e-mt-92cec1b2", "name": "openshift-service-ca.crt", "reconcileID": "d9b0ae2d-b2c0-4c98-b160-01b364fee054", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-92cec1b2"}
2026-07-10T21:38:35Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-92cec1b2"}, "namespace": "ai-tenant-e2e-mt-92cec1b2", "name": "openshift-service-ca.crt", "reconcileID": "d9b0ae2d-b2c0-4c98-b160-01b364fee054", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-92cec1b2"}
2026-07-10T21:38:35Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-92cec1b2"}, "namespace": "ai-tenant-e2e-mt-92cec1b2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e521f5bf-395a-411b-88ba-f08a259bda2c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-92cec1b2"}
2026-07-10T21:38:35Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-92cec1b2"}, "namespace": "ai-tenant-e2e-mt-92cec1b2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e521f5bf-395a-411b-88ba-f08a259bda2c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-92cec1b2"}
2026-07-10T21:38:35Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9701d48a"}, "namespace": "ai-tenant-e2e-mt-9701d48a", "name": "openshift-service-ca.crt", "reconcileID": "38e8135e-c514-4161-a82a-7723a66dc534", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9701d48a"}
2026-07-10T21:38:35Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-9701d48a"}, "namespace": "ai-tenant-e2e-mt-9701d48a", "name": "odh-trusted-ca-bundle", "reconcileID": "ae7ba3cc-1100-488e-9687-02d4df99cd38", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-9701d48a"}
2026-07-10T21:38:35Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9701d48a"}, "namespace": "ai-tenant-e2e-mt-9701d48a", "name": "openshift-service-ca.crt", "reconcileID": "91cdf8b7-9d7d-4d4c-9f13-7d289afb44c5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9701d48a", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:38:35Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9701d48a"}, "namespace": "ai-tenant-e2e-mt-9701d48a", "name": "openshift-service-ca.crt", "reconcileID": "91cdf8b7-9d7d-4d4c-9f13-7d289afb44c5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9701d48a"}
2026-07-10T21:38:35Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9701d48a"}, "namespace": "ai-tenant-e2e-mt-9701d48a", "name": "openshift-service-ca.crt", "reconcileID": "438c724f-6f99-44fb-b82f-0b4fccda2131", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9701d48a", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:38:35Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9701d48a"}, "namespace": "ai-tenant-e2e-mt-9701d48a", "name": "openshift-service-ca.crt", "reconcileID": "438c724f-6f99-44fb-b82f-0b4fccda2131", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T21:38:35Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9701d48a"}, "namespace": "ai-tenant-e2e-mt-9701d48a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dce4a659-827f-4550-b529-288b83582940", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9701d48a", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:38:35Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9701d48a"}, "namespace": "ai-tenant-e2e-mt-9701d48a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dce4a659-827f-4550-b529-288b83582940", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9701d48a"}
2026-07-10T21:38:35Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-9701d48a"}, "namespace": "ai-tenant-e2e-mt-9701d48a", "name": "odh-trusted-ca-bundle", "reconcileID": "919586f2-b67d-4e29-b532-dfa499e6a236", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-9701d48a"}
2026-07-10T21:38:35Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9701d48a"}, "namespace": "ai-tenant-e2e-mt-9701d48a", "name": "openshift-service-ca.crt", "reconcileID": "b3270584-c52b-4838-a7af-c15356da539a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9701d48a"}
2026-07-10T21:38:35Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9701d48a"}, "namespace": "ai-tenant-e2e-mt-9701d48a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "509ed7ff-8166-4260-80f7-a67fe6c9e89a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9701d48a"}
2026-07-10T21:39:12Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9701d48a"}, "namespace": "ai-tenant-e2e-mt-9701d48a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5056e0df-197c-4d7f-8084-259242ec4e3f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9701d48a"}
2026-07-10T21:39:12Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9701d48a"}, "namespace": "ai-tenant-e2e-mt-9701d48a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5056e0df-197c-4d7f-8084-259242ec4e3f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9701d48a", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:39:12Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9701d48a"}, "namespace": "ai-tenant-e2e-mt-9701d48a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5056e0df-197c-4d7f-8084-259242ec4e3f", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-9701d48a because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T21:39:12Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-9701d48a"}, "namespace": "ai-tenant-e2e-mt-9701d48a", "name": "odh-trusted-ca-bundle", "reconcileID": "996a46af-5162-47b2-b0b4-76bfaf3b4c28", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-9701d48a"}
2026-07-10T21:39:12Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-9701d48a"}, "namespace": "ai-tenant-e2e-mt-9701d48a", "name": "odh-trusted-ca-bundle", "reconcileID": "996a46af-5162-47b2-b0b4-76bfaf3b4c28", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-9701d48a"}
2026-07-10T21:39:12Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9701d48a"}, "namespace": "ai-tenant-e2e-mt-9701d48a", "name": "openshift-service-ca.crt", "reconcileID": "e948ee5c-ccd2-4285-803e-5b3d161880d5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9701d48a"}
2026-07-10T21:39:12Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9701d48a"}, "namespace": "ai-tenant-e2e-mt-9701d48a", "name": "openshift-service-ca.crt", "reconcileID": "e948ee5c-ccd2-4285-803e-5b3d161880d5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9701d48a"}
2026-07-10T21:39:12Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9701d48a"}, "namespace": "ai-tenant-e2e-mt-9701d48a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a9310840-4ec3-4cb1-af5b-1f2df55af544", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9701d48a"}
2026-07-10T21:39:12Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9701d48a"}, "namespace": "ai-tenant-e2e-mt-9701d48a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a9310840-4ec3-4cb1-af5b-1f2df55af544", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9701d48a"}
2026-07-10T21:39:13Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-66ee6b","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b", "reconcileID": "41aa8bf7-1279-47d5-a681-fd30aee04f47", "Gateway": "e2e-inf-a-66ee6b", "namespace": "openshift-ingress"}
2026-07-10T21:39:13Z	INFO	Creating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-66ee6b","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b", "reconcileID": "41aa8bf7-1279-47d5-a681-fd30aee04f47", "Gateway": "e2e-inf-a-66ee6b", "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b-authn-ssl"}
2026-07-10T21:39:13Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-66ee6b","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b", "reconcileID": "41aa8bf7-1279-47d5-a681-fd30aee04f47", "Gateway": "e2e-inf-a-66ee6b", "namespace": "openshift-ingress"}
2026-07-10T21:39:13Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-66ee6b","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b", "reconcileID": "e2b3c627-5480-4df9-8d95-aa0f8b0987c6", "Gateway": "e2e-inf-a-66ee6b", "namespace": "openshift-ingress"}
2026-07-10T21:39:13Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-66ee6b","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b", "reconcileID": "e2b3c627-5480-4df9-8d95-aa0f8b0987c6", "Gateway": "e2e-inf-a-66ee6b", "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b-authn-ssl"}
2026-07-10T21:39:13Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-66ee6b","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b", "reconcileID": "e2b3c627-5480-4df9-8d95-aa0f8b0987c6", "Gateway": "e2e-inf-a-66ee6b", "namespace": "openshift-ingress"}
2026-07-10T21:39:13Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-66ee6b","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b", "reconcileID": "3321c242-8533-4c62-b726-e5f6dd594ad9", "Gateway": "e2e-inf-a-66ee6b", "namespace": "openshift-ingress"}
2026-07-10T21:39:13Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-66ee6b","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b", "reconcileID": "3321c242-8533-4c62-b726-e5f6dd594ad9", "Gateway": "e2e-inf-a-66ee6b", "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b-authn-ssl"}
2026-07-10T21:39:13Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-66ee6b","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b", "reconcileID": "3321c242-8533-4c62-b726-e5f6dd594ad9", "Gateway": "e2e-inf-a-66ee6b", "namespace": "openshift-ingress"}
2026-07-10T21:39:13Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-66ee6b","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b", "reconcileID": "c54cf34e-4de9-486e-9bd3-90a92b9bd51a", "Gateway": "e2e-inf-a-66ee6b", "namespace": "openshift-ingress"}
2026-07-10T21:39:13Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-66ee6b","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b", "reconcileID": "c54cf34e-4de9-486e-9bd3-90a92b9bd51a", "Gateway": "e2e-inf-a-66ee6b", "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b-authn-ssl"}
2026-07-10T21:39:13Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-66ee6b","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b", "reconcileID": "c54cf34e-4de9-486e-9bd3-90a92b9bd51a", "Gateway": "e2e-inf-a-66ee6b", "namespace": "openshift-ingress"}
2026-07-10T21:39:13Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-66ee6b","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b", "reconcileID": "eb74c885-752e-46c9-93b1-d7769b8180c7", "Gateway": "e2e-inf-a-66ee6b", "namespace": "openshift-ingress"}
2026-07-10T21:39:13Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-66ee6b","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b", "reconcileID": "eb74c885-752e-46c9-93b1-d7769b8180c7", "Gateway": "e2e-inf-a-66ee6b", "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b-authn-ssl"}
2026-07-10T21:39:13Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-66ee6b","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b", "reconcileID": "eb74c885-752e-46c9-93b1-d7769b8180c7", "Gateway": "e2e-inf-a-66ee6b", "namespace": "openshift-ingress"}
2026-07-10T21:39:14Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-66ee6b","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b", "reconcileID": "23d463eb-5ac5-49e9-bd5e-7f89e77e2003", "Gateway": "e2e-inf-a-66ee6b", "namespace": "openshift-ingress"}
2026-07-10T21:39:14Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-66ee6b","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b", "reconcileID": "23d463eb-5ac5-49e9-bd5e-7f89e77e2003", "Gateway": "e2e-inf-a-66ee6b", "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b-authn-ssl"}
2026-07-10T21:39:14Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-66ee6b","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b", "reconcileID": "23d463eb-5ac5-49e9-bd5e-7f89e77e2003", "Gateway": "e2e-inf-a-66ee6b", "namespace": "openshift-ingress"}
2026-07-10T21:39:15Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-66ee6b","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b", "reconcileID": "4fe35f7f-0ad8-4b15-8615-5eff97b63283", "Gateway": "e2e-inf-a-66ee6b", "namespace": "openshift-ingress"}
2026-07-10T21:39:15Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-66ee6b","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b", "reconcileID": "4fe35f7f-0ad8-4b15-8615-5eff97b63283", "Gateway": "e2e-inf-a-66ee6b", "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b-authn-ssl"}
2026-07-10T21:39:15Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-66ee6b","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b", "reconcileID": "4fe35f7f-0ad8-4b15-8615-5eff97b63283", "Gateway": "e2e-inf-a-66ee6b", "namespace": "openshift-ingress"}
2026-07-10T21:39:17Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-66ee6b"}, "namespace": "ai-tenant-e2e-inf-a-66ee6b", "name": "openshift-service-ca.crt", "reconcileID": "b2b4d4c1-b2dd-4ee5-81e8-1213503b1f52", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-66ee6b"}
2026-07-10T21:39:17Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-66ee6b"}, "namespace": "ai-tenant-e2e-inf-a-66ee6b", "name": "odh-trusted-ca-bundle", "reconcileID": "5b957b9d-ceac-4c30-941a-88c1a9ac958c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-66ee6b"}
2026-07-10T21:39:17Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-66ee6b"}, "namespace": "ai-tenant-e2e-inf-a-66ee6b", "name": "openshift-service-ca.crt", "reconcileID": "9f3cff2e-0cc6-4513-a920-4639f2d3f43b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-66ee6b", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:39:17Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-66ee6b"}, "namespace": "ai-tenant-e2e-inf-a-66ee6b", "name": "openshift-service-ca.crt", "reconcileID": "9f3cff2e-0cc6-4513-a920-4639f2d3f43b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-66ee6b"}
2026-07-10T21:39:17Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-66ee6b"}, "namespace": "ai-tenant-e2e-inf-a-66ee6b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7cdb1706-442e-4753-8c29-6965b91ccc61", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-66ee6b"}
2026-07-10T21:39:17Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-66ee6b"}, "namespace": "ai-tenant-e2e-inf-a-66ee6b", "name": "openshift-service-ca.crt", "reconcileID": "3121f70f-6708-4078-95bf-8a607604bb57", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-66ee6b"}
2026-07-10T21:39:17Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-66ee6b"}, "namespace": "ai-tenant-e2e-inf-a-66ee6b", "name": "odh-trusted-ca-bundle", "reconcileID": "76b1436a-0881-4199-9f53-33ac16f3efb2", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-66ee6b", "update": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:39:17Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-66ee6b"}, "namespace": "ai-tenant-e2e-inf-a-66ee6b", "name": "odh-trusted-ca-bundle", "reconcileID": "76b1436a-0881-4199-9f53-33ac16f3efb2", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-66ee6b"}
2026-07-10T21:39:17Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-66ee6b"}, "namespace": "ai-tenant-e2e-inf-a-66ee6b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9ad87415-790a-4694-8eed-2a823b2b885e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-66ee6b"}
2026-07-10T21:42:32Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-66ee6b"}, "namespace": "ai-tenant-e2e-inf-a-66ee6b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "830597d7-e282-4779-ae07-0595c96bbfe3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-66ee6b"}
2026-07-10T21:42:32Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-66ee6b"}, "namespace": "ai-tenant-e2e-inf-a-66ee6b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "830597d7-e282-4779-ae07-0595c96bbfe3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-66ee6b", "create": "odh-kserve-custom-ca-bundle"}
2026-07-10T21:42:32Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-66ee6b"}, "namespace": "ai-tenant-e2e-inf-a-66ee6b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "830597d7-e282-4779-ae07-0595c96bbfe3", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-inf-a-66ee6b because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-10T21:42:32Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-66ee6b"}, "namespace": "ai-tenant-e2e-inf-a-66ee6b", "name": "odh-trusted-ca-bundle", "reconcileID": "9c85e0ab-a564-4538-b1b8-0c1be68acbd7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-66ee6b"}
2026-07-10T21:42:32Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-66ee6b"}, "namespace": "ai-tenant-e2e-inf-a-66ee6b", "name": "odh-trusted-ca-bundle", "reconcileID": "9c85e0ab-a564-4538-b1b8-0c1be68acbd7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-66ee6b"}
2026-07-10T21:42:32Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-66ee6b"}, "namespace": "ai-tenant-e2e-inf-a-66ee6b", "name": "openshift-service-ca.crt", "reconcileID": "fb39db12-cf28-4af2-b707-7d359c57c35d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-66ee6b"}
2026-07-10T21:42:32Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-66ee6b"}, "namespace": "ai-tenant-e2e-inf-a-66ee6b", "name": "openshift-service-ca.crt", "reconcileID": "fb39db12-cf28-4af2-b707-7d359c57c35d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-66ee6b"}
2026-07-10T21:42:32Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-66ee6b"}, "namespace": "ai-tenant-e2e-inf-a-66ee6b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "59084325-c7a1-499f-9f81-b4b558a12ee4", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-66ee6b"}
2026-07-10T21:42:32Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-66ee6b"}, "namespace": "ai-tenant-e2e-inf-a-66ee6b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "59084325-c7a1-499f-9f81-b4b558a12ee4", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-66ee6b"}
2026-07-10T21:42:33Z	DEBUG	Gateway not found, resources owned by Gateway will be garbage collected	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-66ee6b","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-66ee6b", "reconcileID": "51718e91-472d-4efc-9195-a95ff254452b", "Gateway": "e2e-inf-a-66ee6b", "namespace": "openshift-ingress"}