2026-07-08T17:24:00Z	DEBUG	Starting AuthPolicy reconciliation for LLMInferenceService	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "83d22f57-65c4-40ee-b483-d943819ca442", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"}
2026-07-08T17:24:00Z	DEBUG	Reconciling HTTPRoute AuthPolicy	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "83d22f57-65c4-40ee-b483-d943819ca442", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"}
2026-07-08T17:24:00Z	DEBUG	Processing HTTPRoute AuthPolicy delta	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "83d22f57-65c4-40ee-b483-d943819ca442", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm", "name": "e2e-trlp-test-simulated-kserve-route-authn"}
2026-07-08T17:24:00Z	DEBUG	AuthType	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "83d22f57-65c4-40ee-b483-d943819ca442", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm", "authType": "userdefined"}
2026-07-08T17:24:00Z	DEBUG	Successfully reconciled LLMInferenceService	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "83d22f57-65c4-40ee-b483-d943819ca442", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"}
2026-07-08T17:24:13Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-d38fdc"}, "namespace": "e2e-webhook-test-d38fdc", "name": "openshift-service-ca.crt", "reconcileID": "f7481658-6446-4573-88b1-0fa557d480d3", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-d38fdc"}
2026-07-08T17:24:13Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-d38fdc"}, "namespace": "e2e-webhook-test-d38fdc", "name": "odh-trusted-ca-bundle", "reconcileID": "3b296f9e-106d-4056-8275-10347b26e490", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-d38fdc"}
2026-07-08T17:24:14Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-d38fdc"}, "namespace": "e2e-webhook-test-d38fdc", "name": "openshift-service-ca.crt", "reconcileID": "da7e63ad-4580-4e42-90bf-1d1cdecaabbe", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-d38fdc", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:24:14Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-d38fdc"}, "namespace": "e2e-webhook-test-d38fdc", "name": "openshift-service-ca.crt", "reconcileID": "da7e63ad-4580-4e42-90bf-1d1cdecaabbe", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-d38fdc"}
2026-07-08T17:24:14Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-d38fdc"}, "namespace": "e2e-webhook-test-d38fdc", "name": "openshift-service-ca.crt", "reconcileID": "1fc9c1fe-8fce-4234-9c0f-1c0e5b92e588", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-d38fdc"}
2026-07-08T17:24:14Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-d38fdc"}, "namespace": "e2e-webhook-test-d38fdc", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fbdf40a0-3500-4ef4-8c90-6ca58d1ef26e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-d38fdc"}
2026-07-08T17:24:14Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-d38fdc"}, "namespace": "e2e-webhook-test-d38fdc", "name": "odh-trusted-ca-bundle", "reconcileID": "58dfb984-23ed-44e2-bd95-13060dd1c586", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-d38fdc", "update": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:24:14Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-d38fdc"}, "namespace": "e2e-webhook-test-d38fdc", "name": "odh-trusted-ca-bundle", "reconcileID": "58dfb984-23ed-44e2-bd95-13060dd1c586", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-d38fdc"}
2026-07-08T17:24:14Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-d38fdc"}, "namespace": "e2e-webhook-test-d38fdc", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9f021a66-03e1-4368-9103-48b9a7c2b3df", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-d38fdc"}
2026-07-08T17:24:19Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-d38fdc"}, "namespace": "e2e-webhook-test-d38fdc", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d38e1bd3-d5ad-4c2b-9d7c-83814ce6adac", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-d38fdc"}
2026-07-08T17:24:19Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-d38fdc"}, "namespace": "e2e-webhook-test-d38fdc", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d38e1bd3-d5ad-4c2b-9d7c-83814ce6adac", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-d38fdc", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:24:19Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-d38fdc"}, "namespace": "e2e-webhook-test-d38fdc", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d38e1bd3-d5ad-4c2b-9d7c-83814ce6adac", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-test-d38fdc because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-08T17:24:19Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-d38fdc"}, "namespace": "e2e-webhook-test-d38fdc", "name": "odh-trusted-ca-bundle", "reconcileID": "23bb88bb-0f23-4cee-a7e6-643627529337", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-d38fdc"}
2026-07-08T17:24:19Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-d38fdc"}, "namespace": "e2e-webhook-test-d38fdc", "name": "odh-trusted-ca-bundle", "reconcileID": "23bb88bb-0f23-4cee-a7e6-643627529337", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-d38fdc"}
2026-07-08T17:24:19Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-d38fdc"}, "namespace": "e2e-webhook-test-d38fdc", "name": "openshift-service-ca.crt", "reconcileID": "e266fe6c-a60e-4715-9fd6-82feaaaccf5c", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-d38fdc"}
2026-07-08T17:24:19Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-d38fdc"}, "namespace": "e2e-webhook-test-d38fdc", "name": "openshift-service-ca.crt", "reconcileID": "e266fe6c-a60e-4715-9fd6-82feaaaccf5c", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-d38fdc"}
2026-07-08T17:24:19Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-d38fdc"}, "namespace": "e2e-webhook-test-d38fdc", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "98e0a265-ea73-47fd-a384-e9859ab650f6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-d38fdc"}
2026-07-08T17:24:19Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-d38fdc"}, "namespace": "e2e-webhook-test-d38fdc", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "98e0a265-ea73-47fd-a384-e9859ab650f6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-d38fdc"}
2026-07-08T17:24:20Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-757592"}, "namespace": "e2e-webhook-test-757592", "name": "openshift-service-ca.crt", "reconcileID": "4444c303-d26c-4466-be18-30f923580d04", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-757592"}
2026-07-08T17:24:20Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-757592"}, "namespace": "e2e-webhook-test-757592", "name": "odh-trusted-ca-bundle", "reconcileID": "50994bc8-4911-47f1-ba9c-4a328fee75b4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-757592"}
2026-07-08T17:24:20Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-757592"}, "namespace": "e2e-webhook-test-757592", "name": "openshift-service-ca.crt", "reconcileID": "b11bcd69-3d27-46b1-b5e5-f7ad2367b033", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-757592", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:24:20Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-757592"}, "namespace": "e2e-webhook-test-757592", "name": "openshift-service-ca.crt", "reconcileID": "b11bcd69-3d27-46b1-b5e5-f7ad2367b033", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-757592"}
2026-07-08T17:24:20Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-757592"}, "namespace": "e2e-webhook-test-757592", "name": "openshift-service-ca.crt", "reconcileID": "a3451d08-3c00-491f-8828-693409faa852", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-757592"}
2026-07-08T17:24:20Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-757592"}, "namespace": "e2e-webhook-test-757592", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b4226109-a1e9-4eb6-aff5-6763efaa9d8c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-757592"}
2026-07-08T17:24:20Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-757592"}, "namespace": "e2e-webhook-test-757592", "name": "odh-trusted-ca-bundle", "reconcileID": "0f745d3c-5067-4a09-9f65-76d2e75c0268", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-757592", "update": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:24:20Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-757592"}, "namespace": "e2e-webhook-test-757592", "name": "odh-trusted-ca-bundle", "reconcileID": "0f745d3c-5067-4a09-9f65-76d2e75c0268", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-757592"}
2026-07-08T17:24:20Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-757592"}, "namespace": "e2e-webhook-test-757592", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5516f07d-78d5-438c-b32a-891936477024", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-757592"}
2026-07-08T17:24:25Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-757592"}, "namespace": "e2e-webhook-test-757592", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "698d2ca3-6b41-4363-9f9c-1339b430a128", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-757592"}
2026-07-08T17:24:25Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-757592"}, "namespace": "e2e-webhook-test-757592", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "698d2ca3-6b41-4363-9f9c-1339b430a128", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-757592", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:24:25Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-757592"}, "namespace": "e2e-webhook-test-757592", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "698d2ca3-6b41-4363-9f9c-1339b430a128", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-test-757592 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-08T17:24:25Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-757592"}, "namespace": "e2e-webhook-test-757592", "name": "odh-trusted-ca-bundle", "reconcileID": "e64ff6b6-6c7f-499d-8d58-a1f77139aef0", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-757592"}
2026-07-08T17:24:25Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-757592"}, "namespace": "e2e-webhook-test-757592", "name": "odh-trusted-ca-bundle", "reconcileID": "e64ff6b6-6c7f-499d-8d58-a1f77139aef0", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-757592"}
2026-07-08T17:24:25Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-757592"}, "namespace": "e2e-webhook-test-757592", "name": "openshift-service-ca.crt", "reconcileID": "582f851f-39f2-4957-9c43-0eaf4ebd754c", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-757592"}
2026-07-08T17:24:25Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-757592"}, "namespace": "e2e-webhook-test-757592", "name": "openshift-service-ca.crt", "reconcileID": "582f851f-39f2-4957-9c43-0eaf4ebd754c", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-757592"}
2026-07-08T17:24:25Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-757592"}, "namespace": "e2e-webhook-test-757592", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d02b61a0-c6f7-47bb-b8f9-19af6fa6945d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-757592"}
2026-07-08T17:24:25Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-757592"}, "namespace": "e2e-webhook-test-757592", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d02b61a0-c6f7-47bb-b8f9-19af6fa6945d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-757592"}
2026-07-08T17:33:18Z	INFO	No monitoring namespace detected, skipping monitoring reconciliation.	{"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"}
2026-07-08T17:33:18Z	INFO	Multi Node reconciling.	{"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"}
2026-07-08T17:33:19Z	INFO	ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt	{"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"}
2026-07-08T17:33:19Z	INFO	Multi Node reconciled successfully.	{"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"}
2026-07-08T17:35:08Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "b8d12955-9c58-49df-b71d-12972cd89242", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"}
2026-07-08T17:35:08Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "b8d12955-9c58-49df-b71d-12972cd89242", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"}
2026-07-08T17:35:08Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "b8d12955-9c58-49df-b71d-12972cd89242", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"}
2026-07-08T17:35:15Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "5a6c0f07-eeca-4486-b62e-7154ec7effb7", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"}
2026-07-08T17:35:15Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "5a6c0f07-eeca-4486-b62e-7154ec7effb7", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"}
2026-07-08T17:35:15Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "5a6c0f07-eeca-4486-b62e-7154ec7effb7", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"}
2026-07-08T17:35:23Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "f5096ffa-2386-492c-90bf-9eee61ee7bde", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"}
2026-07-08T17:35:23Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "f5096ffa-2386-492c-90bf-9eee61ee7bde", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"}
2026-07-08T17:35:23Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "f5096ffa-2386-492c-90bf-9eee61ee7bde", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"}
2026-07-08T17:35:28Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "8f649ac0-cec9-4218-be81-b475fc35c548", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"}
2026-07-08T17:35:28Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "8f649ac0-cec9-4218-be81-b475fc35c548", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"}
2026-07-08T17:35:29Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "8f649ac0-cec9-4218-be81-b475fc35c548", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"}
2026-07-08T17:35:31Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-ait-wrong-d9dd4026"}, "namespace": "e2e-ait-wrong-d9dd4026", "name": "openshift-service-ca.crt", "reconcileID": "f6f2ea56-deba-4946-aef7-640a21619565", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-ait-wrong-d9dd4026"}
2026-07-08T17:35:31Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-ait-wrong-d9dd4026"}, "namespace": "e2e-ait-wrong-d9dd4026", "name": "odh-trusted-ca-bundle", "reconcileID": "67779a58-549a-418c-a1f3-4618cec77e5e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-ait-wrong-d9dd4026"}
2026-07-08T17:35:31Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-ait-wrong-d9dd4026"}, "namespace": "e2e-ait-wrong-d9dd4026", "name": "openshift-service-ca.crt", "reconcileID": "1b74d399-a93c-44b9-ad09-8ff5c453ac42", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-ait-wrong-d9dd4026", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:35:31Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-ait-wrong-d9dd4026"}, "namespace": "e2e-ait-wrong-d9dd4026", "name": "openshift-service-ca.crt", "reconcileID": "1b74d399-a93c-44b9-ad09-8ff5c453ac42", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-ait-wrong-d9dd4026"}
2026-07-08T17:35:31Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-ait-wrong-d9dd4026"}, "namespace": "e2e-ait-wrong-d9dd4026", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5d323562-53ae-4f8f-9269-bc951fc34168", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-ait-wrong-d9dd4026"}
2026-07-08T17:35:31Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-ait-wrong-d9dd4026"}, "namespace": "e2e-ait-wrong-d9dd4026", "name": "openshift-service-ca.crt", "reconcileID": "45efee13-0724-4922-b0ac-973195b049e8", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-ait-wrong-d9dd4026"}
2026-07-08T17:35:31Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-ait-wrong-d9dd4026"}, "namespace": "e2e-ait-wrong-d9dd4026", "name": "odh-trusted-ca-bundle", "reconcileID": "5ae7d17a-9d63-4789-b3d7-df9a154d5db6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-ait-wrong-d9dd4026", "update": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:35:31Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-ait-wrong-d9dd4026"}, "namespace": "e2e-ait-wrong-d9dd4026", "name": "odh-trusted-ca-bundle", "reconcileID": "5ae7d17a-9d63-4789-b3d7-df9a154d5db6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-ait-wrong-d9dd4026"}
2026-07-08T17:35:31Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-ait-wrong-d9dd4026"}, "namespace": "e2e-ait-wrong-d9dd4026", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "67bfe5f4-ff43-4623-9dde-e7a2e02d3cbf", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-ait-wrong-d9dd4026"}
2026-07-08T17:35:37Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-ait-wrong-d9dd4026"}, "namespace": "e2e-ait-wrong-d9dd4026", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a0c14204-182e-44b9-9240-81f2a98d3b41", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-ait-wrong-d9dd4026"}
2026-07-08T17:35:37Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-ait-wrong-d9dd4026"}, "namespace": "e2e-ait-wrong-d9dd4026", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a0c14204-182e-44b9-9240-81f2a98d3b41", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-ait-wrong-d9dd4026", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:35:37Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-ait-wrong-d9dd4026"}, "namespace": "e2e-ait-wrong-d9dd4026", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a0c14204-182e-44b9-9240-81f2a98d3b41", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-ait-wrong-d9dd4026 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-08T17:35:37Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-ait-wrong-d9dd4026"}, "namespace": "e2e-ait-wrong-d9dd4026", "name": "odh-trusted-ca-bundle", "reconcileID": "67a91a9b-c34d-41b3-8ee9-e3de74aecfa6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-ait-wrong-d9dd4026"}
2026-07-08T17:35:37Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-ait-wrong-d9dd4026"}, "namespace": "e2e-ait-wrong-d9dd4026", "name": "odh-trusted-ca-bundle", "reconcileID": "67a91a9b-c34d-41b3-8ee9-e3de74aecfa6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-ait-wrong-d9dd4026"}
2026-07-08T17:35:37Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-ait-wrong-d9dd4026"}, "namespace": "e2e-ait-wrong-d9dd4026", "name": "openshift-service-ca.crt", "reconcileID": "0b906868-eec7-4889-a6a5-dcfd166631da", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-ait-wrong-d9dd4026"}
2026-07-08T17:35:37Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-ait-wrong-d9dd4026"}, "namespace": "e2e-ait-wrong-d9dd4026", "name": "openshift-service-ca.crt", "reconcileID": "0b906868-eec7-4889-a6a5-dcfd166631da", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-ait-wrong-d9dd4026"}
2026-07-08T17:35:37Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-ait-wrong-d9dd4026"}, "namespace": "e2e-ait-wrong-d9dd4026", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dff3e8c4-fcf0-4ae9-9d06-c32df32fd8b1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-ait-wrong-d9dd4026"}
2026-07-08T17:35:37Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-ait-wrong-d9dd4026"}, "namespace": "e2e-ait-wrong-d9dd4026", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dff3e8c4-fcf0-4ae9-9d06-c32df32fd8b1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-ait-wrong-d9dd4026"}
2026-07-08T17:35:38Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-27f79031","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-27f79031", "reconcileID": "ea01714b-5469-4e81-a1cf-6339b0e4bb47", "Gateway": "e2e-ait-27f79031", "namespace": "openshift-ingress"}
2026-07-08T17:35:38Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-27f79031","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-27f79031", "reconcileID": "f4fbaed2-7928-4607-8662-b37e3be44e6e", "Gateway": "e2e-ait-27f79031", "namespace": "openshift-ingress"}
2026-07-08T17:35:38Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-27f79031","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-27f79031", "reconcileID": "eff4a54b-19c5-4cac-a259-92bf1db63083", "Gateway": "e2e-ait-27f79031", "namespace": "openshift-ingress"}
2026-07-08T17:35:38Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-27f79031","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-27f79031", "reconcileID": "fc300d49-b1e0-4913-8e8d-da0db9cf6f7c", "Gateway": "e2e-ait-27f79031", "namespace": "openshift-ingress"}
2026-07-08T17:35:38Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-27f79031","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-27f79031", "reconcileID": "706e0fbc-8864-4833-80c8-9144fb7ad894", "Gateway": "e2e-ait-27f79031", "namespace": "openshift-ingress"}
2026-07-08T17:35:39Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-27f79031"}, "namespace": "ai-tenant-e2e-ait-27f79031", "name": "openshift-service-ca.crt", "reconcileID": "c7db30b6-99d8-433c-a728-f10cd25a03c7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-27f79031"}
2026-07-08T17:35:39Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-27f79031"}, "namespace": "ai-tenant-e2e-ait-27f79031", "name": "odh-trusted-ca-bundle", "reconcileID": "8157ae09-f765-418d-a641-b66323d94eba", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-27f79031"}
2026-07-08T17:35:39Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-27f79031"}, "namespace": "ai-tenant-e2e-ait-27f79031", "name": "openshift-service-ca.crt", "reconcileID": "2d8c5059-2db7-49e7-a083-381938975461", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-27f79031", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:35:39Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-27f79031"}, "namespace": "ai-tenant-e2e-ait-27f79031", "name": "openshift-service-ca.crt", "reconcileID": "2d8c5059-2db7-49e7-a083-381938975461", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-27f79031"}
2026-07-08T17:35:39Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-27f79031"}, "namespace": "ai-tenant-e2e-ait-27f79031", "name": "openshift-service-ca.crt", "reconcileID": "de4db9fd-7f28-4b35-99bd-552dff9db673", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-27f79031"}
2026-07-08T17:35:39Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-27f79031"}, "namespace": "ai-tenant-e2e-ait-27f79031", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9fc62831-7fd5-4b6d-9932-7715b2ff6a11", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-27f79031"}
2026-07-08T17:35:39Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-27f79031","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-27f79031", "reconcileID": "ad19affc-d497-401f-af3e-fd0f84c68ae2", "Gateway": "e2e-ait-27f79031", "namespace": "openshift-ingress"}
2026-07-08T17:35:39Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-27f79031"}, "namespace": "ai-tenant-e2e-ait-27f79031", "name": "odh-trusted-ca-bundle", "reconcileID": "71bf840d-d2cb-448c-9427-c720826aa948", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-27f79031", "update": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:35:39Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-27f79031"}, "namespace": "ai-tenant-e2e-ait-27f79031", "name": "odh-trusted-ca-bundle", "reconcileID": "71bf840d-d2cb-448c-9427-c720826aa948", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-27f79031"}
2026-07-08T17:35:39Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-27f79031"}, "namespace": "ai-tenant-e2e-ait-27f79031", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "283ea085-a2cd-44ff-929f-b373db29a517", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-27f79031"}
2026-07-08T17:35:40Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "c7fe672b-06e2-4bb9-9f36-a842fd414e02", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"}
2026-07-08T17:35:42Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-27f79031","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-27f79031", "reconcileID": "a7de31fe-7033-489b-af69-7b7fba99f688", "Gateway": "e2e-ait-27f79031", "namespace": "openshift-ingress"}
2026-07-08T17:35:50Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-27f79031","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-27f79031", "reconcileID": "b9ae0674-9500-4e28-b429-5248419c2a70", "Gateway": "e2e-ait-27f79031", "namespace": "openshift-ingress"}
2026-07-08T17:36:49Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-27f79031"}, "namespace": "ai-tenant-e2e-ait-27f79031", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f7eec8a9-e820-4e9e-9aec-ceecdf822a66", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-27f79031"}
2026-07-08T17:36:49Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-27f79031"}, "namespace": "ai-tenant-e2e-ait-27f79031", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f7eec8a9-e820-4e9e-9aec-ceecdf822a66", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-27f79031", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:36:49Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-27f79031"}, "namespace": "ai-tenant-e2e-ait-27f79031", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f7eec8a9-e820-4e9e-9aec-ceecdf822a66", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ait-27f79031 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-08T17:36:49Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-27f79031"}, "namespace": "ai-tenant-e2e-ait-27f79031", "name": "odh-trusted-ca-bundle", "reconcileID": "d1ad344a-65f8-49ca-a128-0029e66f063c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-27f79031"}
2026-07-08T17:36:49Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-27f79031"}, "namespace": "ai-tenant-e2e-ait-27f79031", "name": "odh-trusted-ca-bundle", "reconcileID": "d1ad344a-65f8-49ca-a128-0029e66f063c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-27f79031"}
2026-07-08T17:36:49Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-27f79031"}, "namespace": "ai-tenant-e2e-ait-27f79031", "name": "openshift-service-ca.crt", "reconcileID": "9fe3f2cf-bd63-4cdb-b34f-51dd281cdae6", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-27f79031"}
2026-07-08T17:36:49Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-27f79031"}, "namespace": "ai-tenant-e2e-ait-27f79031", "name": "openshift-service-ca.crt", "reconcileID": "9fe3f2cf-bd63-4cdb-b34f-51dd281cdae6", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-27f79031"}
2026-07-08T17:36:49Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-27f79031"}, "namespace": "ai-tenant-e2e-ait-27f79031", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "da9a9f0f-fc5a-4f9a-9ee6-c6e6753e7f6c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-27f79031"}
2026-07-08T17:36:49Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-27f79031"}, "namespace": "ai-tenant-e2e-ait-27f79031", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "da9a9f0f-fc5a-4f9a-9ee6-c6e6753e7f6c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-27f79031"}
2026-07-08T17:36:56Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-220a4915","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-220a4915", "reconcileID": "d8e11b0f-fd47-4684-8ad2-ee2f7d991e7a", "Gateway": "e2e-ait-220a4915", "namespace": "openshift-ingress"}
2026-07-08T17:36:56Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-220a4915","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-220a4915", "reconcileID": "2e006c81-0fd2-469e-a3ca-5e1d24af4e97", "Gateway": "e2e-ait-220a4915", "namespace": "openshift-ingress"}
2026-07-08T17:36:56Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-220a4915","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-220a4915", "reconcileID": "64124da9-3335-4186-a73d-54ccc2983fc7", "Gateway": "e2e-ait-220a4915", "namespace": "openshift-ingress"}
2026-07-08T17:36:56Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-220a4915","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-220a4915", "reconcileID": "7d6510a2-4506-4f6a-9420-b4af0277b8c9", "Gateway": "e2e-ait-220a4915", "namespace": "openshift-ingress"}
2026-07-08T17:36:56Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-220a4915","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-220a4915", "reconcileID": "ae917a64-95c3-4ef2-b657-1e96a9541a0f", "Gateway": "e2e-ait-220a4915", "namespace": "openshift-ingress"}
2026-07-08T17:36:57Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-220a4915","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-220a4915", "reconcileID": "f9d936d7-7e58-4db9-9162-15c53ef1aa19", "Gateway": "e2e-ait-220a4915", "namespace": "openshift-ingress"}
2026-07-08T17:37:00Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-220a4915","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-220a4915", "reconcileID": "32261f51-fb89-4a19-85c2-399f58ea6944", "Gateway": "e2e-ait-220a4915", "namespace": "openshift-ingress"}
2026-07-08T17:38:53Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "8c1f5404-a437-4a71-a2ce-c38951053c86", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"}
2026-07-08T17:40:01Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-9a3a2f79","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-9a3a2f79", "reconcileID": "ccac24a2-bb2d-4f32-8fb8-e066158bffe6", "Gateway": "e2e-derive-9a3a2f79", "namespace": "openshift-ingress"}
2026-07-08T17:40:01Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-9a3a2f79","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-9a3a2f79", "reconcileID": "4c7862ef-c632-4ffe-8a93-a9e430af84da", "Gateway": "e2e-derive-9a3a2f79", "namespace": "openshift-ingress"}
2026-07-08T17:40:01Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-9a3a2f79","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-9a3a2f79", "reconcileID": "23f9d333-4fdd-446b-8942-410899d47487", "Gateway": "e2e-derive-9a3a2f79", "namespace": "openshift-ingress"}
2026-07-08T17:40:01Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-9a3a2f79","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-9a3a2f79", "reconcileID": "89546114-0314-4ca9-9485-3d7cb88ad2ee", "Gateway": "e2e-derive-9a3a2f79", "namespace": "openshift-ingress"}
2026-07-08T17:40:01Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-9a3a2f79","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-9a3a2f79", "reconcileID": "4731a125-e779-4e5e-8c8c-cc77ca30f586", "Gateway": "e2e-derive-9a3a2f79", "namespace": "openshift-ingress"}
2026-07-08T17:40:02Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-9a3a2f79","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-9a3a2f79", "reconcileID": "6a490a69-3e5c-4a4b-9e4d-f074ddbc6858", "Gateway": "e2e-derive-9a3a2f79", "namespace": "openshift-ingress"}
2026-07-08T17:40:16Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-9a3a2f79","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-9a3a2f79", "reconcileID": "c78b062c-e7ba-408c-af81-b428198dcf53", "Gateway": "e2e-derive-9a3a2f79", "namespace": "openshift-ingress"}
2026-07-08T17:40:57Z	INFO	No monitoring namespace detected, skipping monitoring reconciliation.	{"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"}
2026-07-08T17:40:57Z	INFO	Multi Node reconciling.	{"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"}
2026-07-08T17:41:00Z	INFO	ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt	{"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"}
2026-07-08T17:41:00Z	INFO	Multi Node reconciled successfully.	{"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"}
2026-07-08T17:42:05Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-34cb76e4"}, "namespace": "ai-tenant-e2e-mt-34cb76e4", "name": "openshift-service-ca.crt", "reconcileID": "5b12da45-7e9a-4cf0-ac66-679d43c734f6", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-34cb76e4"}
2026-07-08T17:42:05Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-34cb76e4"}, "namespace": "ai-tenant-e2e-mt-34cb76e4", "name": "odh-trusted-ca-bundle", "reconcileID": "938ab21b-c8eb-4276-a908-e9caea8f2b0f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-34cb76e4"}
2026-07-08T17:42:05Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-34cb76e4"}, "namespace": "ai-tenant-e2e-mt-34cb76e4", "name": "openshift-service-ca.crt", "reconcileID": "6bf8608a-5ac0-40fa-9c0b-e2064248e44f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-34cb76e4", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:42:05Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-34cb76e4"}, "namespace": "ai-tenant-e2e-mt-34cb76e4", "name": "openshift-service-ca.crt", "reconcileID": "6bf8608a-5ac0-40fa-9c0b-e2064248e44f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-34cb76e4"}
2026-07-08T17:42:05Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-34cb76e4"}, "namespace": "ai-tenant-e2e-mt-34cb76e4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6dbcab7e-f540-4c2b-816a-2cab3dd19aa9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-34cb76e4"}
2026-07-08T17:42:05Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-34cb76e4"}, "namespace": "ai-tenant-e2e-mt-34cb76e4", "name": "openshift-service-ca.crt", "reconcileID": "d082932d-7029-41fa-9996-e13c0fe73ece", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-34cb76e4"}
2026-07-08T17:42:05Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-34cb76e4"}, "namespace": "ai-tenant-e2e-mt-34cb76e4", "name": "odh-trusted-ca-bundle", "reconcileID": "8595e2d0-cff4-42d8-a012-d31deb376d44", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-34cb76e4", "update": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:42:05Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-34cb76e4"}, "namespace": "ai-tenant-e2e-mt-34cb76e4", "name": "odh-trusted-ca-bundle", "reconcileID": "8595e2d0-cff4-42d8-a012-d31deb376d44", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-34cb76e4"}
2026-07-08T17:42:05Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-34cb76e4"}, "namespace": "ai-tenant-e2e-mt-34cb76e4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "17b139ac-7ed7-4a36-860d-281389bd47a0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-34cb76e4"}
2026-07-08T17:42:13Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-34cb76e4"}, "namespace": "ai-tenant-e2e-mt-34cb76e4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "23b72b69-d209-4ba5-aad4-0bc5611e5d9b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-34cb76e4"}
2026-07-08T17:42:13Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-34cb76e4"}, "namespace": "ai-tenant-e2e-mt-34cb76e4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "23b72b69-d209-4ba5-aad4-0bc5611e5d9b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-34cb76e4", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:42:13Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-34cb76e4"}, "namespace": "ai-tenant-e2e-mt-34cb76e4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "23b72b69-d209-4ba5-aad4-0bc5611e5d9b", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-34cb76e4 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-08T17:42:13Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-34cb76e4"}, "namespace": "ai-tenant-e2e-mt-34cb76e4", "name": "odh-trusted-ca-bundle", "reconcileID": "879b6ef4-ec3e-4634-9fb8-23673d47d4e2", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-34cb76e4"}
2026-07-08T17:42:13Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-34cb76e4"}, "namespace": "ai-tenant-e2e-mt-34cb76e4", "name": "odh-trusted-ca-bundle", "reconcileID": "879b6ef4-ec3e-4634-9fb8-23673d47d4e2", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-34cb76e4"}
2026-07-08T17:42:13Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-34cb76e4"}, "namespace": "ai-tenant-e2e-mt-34cb76e4", "name": "openshift-service-ca.crt", "reconcileID": "29d011a7-b5bf-4b27-925a-3ab7dbf592b1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-34cb76e4"}
2026-07-08T17:42:13Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-34cb76e4"}, "namespace": "ai-tenant-e2e-mt-34cb76e4", "name": "openshift-service-ca.crt", "reconcileID": "29d011a7-b5bf-4b27-925a-3ab7dbf592b1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-34cb76e4"}
2026-07-08T17:42:13Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-34cb76e4"}, "namespace": "ai-tenant-e2e-mt-34cb76e4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cdfe5086-42bc-4390-bdc7-accefc88138d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-34cb76e4"}
2026-07-08T17:42:13Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-34cb76e4"}, "namespace": "ai-tenant-e2e-mt-34cb76e4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cdfe5086-42bc-4390-bdc7-accefc88138d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-34cb76e4"}
2026-07-08T17:42:13Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-ef3a305b"}, "namespace": "ai-tenant-e2e-mt-ef3a305b", "name": "openshift-service-ca.crt", "reconcileID": "9fd012cd-ca4a-4fd5-9ec2-dd1c6c67755d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-ef3a305b"}
2026-07-08T17:42:13Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-ef3a305b"}, "namespace": "ai-tenant-e2e-mt-ef3a305b", "name": "odh-trusted-ca-bundle", "reconcileID": "e1c74ab1-02f8-4ceb-b847-fce9d41ca5f0", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-ef3a305b"}
2026-07-08T17:42:13Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-ef3a305b"}, "namespace": "ai-tenant-e2e-mt-ef3a305b", "name": "openshift-service-ca.crt", "reconcileID": "5168d530-f386-455b-a59f-fde5aa613edc", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-ef3a305b", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:42:13Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-ef3a305b"}, "namespace": "ai-tenant-e2e-mt-ef3a305b", "name": "openshift-service-ca.crt", "reconcileID": "5168d530-f386-455b-a59f-fde5aa613edc", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-ef3a305b"}
2026-07-08T17:42:13Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-ef3a305b"}, "namespace": "ai-tenant-e2e-mt-ef3a305b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "25f851a2-ffe5-4b4d-ad98-190403b4e953", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-ef3a305b"}
2026-07-08T17:42:13Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-ef3a305b"}, "namespace": "ai-tenant-e2e-mt-ef3a305b", "name": "openshift-service-ca.crt", "reconcileID": "a5faba0a-f619-4d87-abf0-59ab1754d9a2", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-ef3a305b"}
2026-07-08T17:42:13Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-ef3a305b"}, "namespace": "ai-tenant-e2e-mt-ef3a305b", "name": "odh-trusted-ca-bundle", "reconcileID": "f56a7c33-055b-4218-bc51-78e63e961e0b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-ef3a305b", "update": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:42:13Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-ef3a305b"}, "namespace": "ai-tenant-e2e-mt-ef3a305b", "name": "odh-trusted-ca-bundle", "reconcileID": "f56a7c33-055b-4218-bc51-78e63e961e0b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-ef3a305b"}
2026-07-08T17:42:13Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-ef3a305b"}, "namespace": "ai-tenant-e2e-mt-ef3a305b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d1ae9113-0056-424c-9547-fd86f7eb4c80", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-ef3a305b"}
2026-07-08T17:42:45Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-ef3a305b"}, "namespace": "ai-tenant-e2e-mt-ef3a305b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "67434341-c483-4a8b-933c-565d3b7dfb65", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-ef3a305b"}
2026-07-08T17:42:45Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-ef3a305b"}, "namespace": "ai-tenant-e2e-mt-ef3a305b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "67434341-c483-4a8b-933c-565d3b7dfb65", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-ef3a305b", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:42:45Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-ef3a305b"}, "namespace": "ai-tenant-e2e-mt-ef3a305b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "67434341-c483-4a8b-933c-565d3b7dfb65", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-ef3a305b because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-08T17:42:45Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-ef3a305b"}, "namespace": "ai-tenant-e2e-mt-ef3a305b", "name": "odh-trusted-ca-bundle", "reconcileID": "562a7ee2-d8cd-4052-9210-85c2eaf019c3", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-ef3a305b"}
2026-07-08T17:42:45Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-ef3a305b"}, "namespace": "ai-tenant-e2e-mt-ef3a305b", "name": "odh-trusted-ca-bundle", "reconcileID": "562a7ee2-d8cd-4052-9210-85c2eaf019c3", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-ef3a305b"}
2026-07-08T17:42:45Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-ef3a305b"}, "namespace": "ai-tenant-e2e-mt-ef3a305b", "name": "openshift-service-ca.crt", "reconcileID": "acfd1d41-66df-4930-ab59-0bf2a5afe2dc", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-ef3a305b"}
2026-07-08T17:42:45Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-ef3a305b"}, "namespace": "ai-tenant-e2e-mt-ef3a305b", "name": "openshift-service-ca.crt", "reconcileID": "acfd1d41-66df-4930-ab59-0bf2a5afe2dc", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-ef3a305b"}
2026-07-08T17:42:45Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-ef3a305b"}, "namespace": "ai-tenant-e2e-mt-ef3a305b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "386b47b5-3224-47f7-b0c8-62e04dc9053c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-ef3a305b"}
2026-07-08T17:42:45Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-ef3a305b"}, "namespace": "ai-tenant-e2e-mt-ef3a305b", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "386b47b5-3224-47f7-b0c8-62e04dc9053c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-ef3a305b"}
2026-07-08T17:42:45Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-83e1305a"}, "namespace": "e2e-unlabeled-83e1305a", "name": "openshift-service-ca.crt", "reconcileID": "3245b72e-fda6-4f79-a04b-3b563f7b5c65", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-83e1305a"}
2026-07-08T17:42:45Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-83e1305a"}, "namespace": "e2e-unlabeled-83e1305a", "name": "odh-trusted-ca-bundle", "reconcileID": "930f6b31-3c49-4285-85e9-0a718546dce9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-83e1305a"}
2026-07-08T17:42:45Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-83e1305a"}, "namespace": "e2e-unlabeled-83e1305a", "name": "openshift-service-ca.crt", "reconcileID": "19e8a0b8-9798-4d8c-8395-c7a83156abe0", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-83e1305a", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:42:45Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-83e1305a"}, "namespace": "e2e-unlabeled-83e1305a", "name": "openshift-service-ca.crt", "reconcileID": "19e8a0b8-9798-4d8c-8395-c7a83156abe0", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-83e1305a"}
2026-07-08T17:42:45Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-83e1305a"}, "namespace": "e2e-unlabeled-83e1305a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "de198cc8-fc69-4c84-8c0e-7950a355d528", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-83e1305a"}
2026-07-08T17:42:45Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-83e1305a"}, "namespace": "e2e-unlabeled-83e1305a", "name": "openshift-service-ca.crt", "reconcileID": "ba2b1847-2c76-4516-a24f-17d3b606b591", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-83e1305a"}
2026-07-08T17:42:45Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-83e1305a"}, "namespace": "e2e-unlabeled-83e1305a", "name": "odh-trusted-ca-bundle", "reconcileID": "27ae51eb-2c34-49f8-93f3-fb4e7773f068", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-83e1305a", "update": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:42:45Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-83e1305a"}, "namespace": "e2e-unlabeled-83e1305a", "name": "odh-trusted-ca-bundle", "reconcileID": "27ae51eb-2c34-49f8-93f3-fb4e7773f068", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-83e1305a"}
2026-07-08T17:42:45Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-83e1305a"}, "namespace": "e2e-unlabeled-83e1305a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dc1de0ab-29a9-47a9-ae11-0a6b2a7cdd2c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-83e1305a"}
2026-07-08T17:43:06Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-83e1305a"}, "namespace": "e2e-unlabeled-83e1305a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1f47d474-a5bb-449d-bf4b-d0f80bbe8f13", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-83e1305a"}
2026-07-08T17:43:06Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-83e1305a"}, "namespace": "e2e-unlabeled-83e1305a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1f47d474-a5bb-449d-bf4b-d0f80bbe8f13", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-83e1305a", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:43:06Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-83e1305a"}, "namespace": "e2e-unlabeled-83e1305a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1f47d474-a5bb-449d-bf4b-d0f80bbe8f13", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-unlabeled-83e1305a because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-08T17:43:06Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-83e1305a"}, "namespace": "e2e-unlabeled-83e1305a", "name": "odh-trusted-ca-bundle", "reconcileID": "6efacbba-8df5-43f4-8212-a8c6fc5be430", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-83e1305a"}
2026-07-08T17:43:06Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-83e1305a"}, "namespace": "e2e-unlabeled-83e1305a", "name": "odh-trusted-ca-bundle", "reconcileID": "6efacbba-8df5-43f4-8212-a8c6fc5be430", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-83e1305a"}
2026-07-08T17:43:06Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-83e1305a"}, "namespace": "e2e-unlabeled-83e1305a", "name": "openshift-service-ca.crt", "reconcileID": "e985ff08-0a47-47cc-a5fd-d8b9b7b62dff", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-83e1305a"}
2026-07-08T17:43:06Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-83e1305a"}, "namespace": "e2e-unlabeled-83e1305a", "name": "openshift-service-ca.crt", "reconcileID": "e985ff08-0a47-47cc-a5fd-d8b9b7b62dff", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-83e1305a"}
2026-07-08T17:43:06Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-83e1305a"}, "namespace": "e2e-unlabeled-83e1305a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8584b933-57cd-4b64-a1eb-ac84e310bf01", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-83e1305a"}
2026-07-08T17:43:06Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-83e1305a"}, "namespace": "e2e-unlabeled-83e1305a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8584b933-57cd-4b64-a1eb-ac84e310bf01", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-83e1305a"}
2026-07-08T17:43:07Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-301c9bc8"}, "namespace": "ai-tenant-e2e-mt-301c9bc8", "name": "openshift-service-ca.crt", "reconcileID": "39970dfc-eff7-4d97-b131-cdcc8cff1ea5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-301c9bc8"}
2026-07-08T17:43:07Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-301c9bc8"}, "namespace": "ai-tenant-e2e-mt-301c9bc8", "name": "odh-trusted-ca-bundle", "reconcileID": "2af62176-85f9-4774-864a-3862f8747727", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-301c9bc8"}
2026-07-08T17:43:07Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-301c9bc8"}, "namespace": "ai-tenant-e2e-mt-301c9bc8", "name": "openshift-service-ca.crt", "reconcileID": "318fc115-b4fc-402b-89c6-133b34e51617", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-301c9bc8", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:43:07Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-301c9bc8"}, "namespace": "ai-tenant-e2e-mt-301c9bc8", "name": "openshift-service-ca.crt", "reconcileID": "318fc115-b4fc-402b-89c6-133b34e51617", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-301c9bc8"}
2026-07-08T17:43:07Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-301c9bc8"}, "namespace": "ai-tenant-e2e-mt-301c9bc8", "name": "openshift-service-ca.crt", "reconcileID": "93d3fabc-4790-4cb9-aef4-5430fdc91f9b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-301c9bc8"}
2026-07-08T17:43:07Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-301c9bc8"}, "namespace": "ai-tenant-e2e-mt-301c9bc8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "86750249-5feb-48f9-8b59-fafc147ed29c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-301c9bc8"}
2026-07-08T17:43:07Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-301c9bc8"}, "namespace": "ai-tenant-e2e-mt-301c9bc8", "name": "odh-trusted-ca-bundle", "reconcileID": "5b586d82-2bb3-41bd-be9c-f4455ad0a7ee", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-301c9bc8", "update": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:43:07Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-301c9bc8"}, "namespace": "ai-tenant-e2e-mt-301c9bc8", "name": "odh-trusted-ca-bundle", "reconcileID": "5b586d82-2bb3-41bd-be9c-f4455ad0a7ee", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-301c9bc8"}
2026-07-08T17:43:07Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-301c9bc8"}, "namespace": "ai-tenant-e2e-mt-301c9bc8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b98fc270-52d0-4d56-b9df-df96f6800a4b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-301c9bc8"}
2026-07-08T17:43:24Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-301c9bc8"}, "namespace": "ai-tenant-e2e-mt-301c9bc8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5418c868-840e-4414-82c7-4b9d21d12cd5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-301c9bc8"}
2026-07-08T17:43:24Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-301c9bc8"}, "namespace": "ai-tenant-e2e-mt-301c9bc8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5418c868-840e-4414-82c7-4b9d21d12cd5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-301c9bc8", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:43:24Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-301c9bc8"}, "namespace": "ai-tenant-e2e-mt-301c9bc8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5418c868-840e-4414-82c7-4b9d21d12cd5", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-301c9bc8 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-08T17:43:24Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-301c9bc8"}, "namespace": "ai-tenant-e2e-mt-301c9bc8", "name": "odh-trusted-ca-bundle", "reconcileID": "89d3e7c7-ce88-4ca7-bbac-730588a43e3f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-301c9bc8"}
2026-07-08T17:43:24Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-301c9bc8"}, "namespace": "ai-tenant-e2e-mt-301c9bc8", "name": "odh-trusted-ca-bundle", "reconcileID": "89d3e7c7-ce88-4ca7-bbac-730588a43e3f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-301c9bc8"}
2026-07-08T17:43:24Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-301c9bc8"}, "namespace": "ai-tenant-e2e-mt-301c9bc8", "name": "openshift-service-ca.crt", "reconcileID": "14716fcb-0381-4fc3-8e5a-d210256e48e5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-301c9bc8"}
2026-07-08T17:43:24Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-301c9bc8"}, "namespace": "ai-tenant-e2e-mt-301c9bc8", "name": "openshift-service-ca.crt", "reconcileID": "14716fcb-0381-4fc3-8e5a-d210256e48e5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-301c9bc8"}
2026-07-08T17:43:24Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-301c9bc8"}, "namespace": "ai-tenant-e2e-mt-301c9bc8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e1881196-d564-417f-9f21-e7b5afa3de0e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-301c9bc8"}
2026-07-08T17:43:24Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-301c9bc8"}, "namespace": "ai-tenant-e2e-mt-301c9bc8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e1881196-d564-417f-9f21-e7b5afa3de0e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-301c9bc8"}
2026-07-08T17:43:25Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-edd1beb3"}, "namespace": "ai-tenant-e2e-mt-edd1beb3", "name": "openshift-service-ca.crt", "reconcileID": "cb00838f-430c-410e-a9b0-9630aec04ea4", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-edd1beb3"}
2026-07-08T17:43:25Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-edd1beb3"}, "namespace": "ai-tenant-e2e-mt-edd1beb3", "name": "odh-trusted-ca-bundle", "reconcileID": "84b49eee-2dd4-4593-9182-39be9009aa9d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-edd1beb3"}
2026-07-08T17:43:25Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-edd1beb3"}, "namespace": "ai-tenant-e2e-mt-edd1beb3", "name": "openshift-service-ca.crt", "reconcileID": "5992c214-e1bf-4b9d-862e-4e499de49740", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-edd1beb3", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:43:25Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-edd1beb3"}, "namespace": "ai-tenant-e2e-mt-edd1beb3", "name": "openshift-service-ca.crt", "reconcileID": "5992c214-e1bf-4b9d-862e-4e499de49740", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-edd1beb3"}
2026-07-08T17:43:25Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-edd1beb3"}, "namespace": "ai-tenant-e2e-mt-edd1beb3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e5dda9e8-5bd8-4c4f-b3d0-6b47be0d5697", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-edd1beb3"}
2026-07-08T17:43:25Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-edd1beb3"}, "namespace": "ai-tenant-e2e-mt-edd1beb3", "name": "openshift-service-ca.crt", "reconcileID": "ec8f3300-22b6-4693-aa94-a4b802a856d3", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-edd1beb3"}
2026-07-08T17:43:25Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-edd1beb3"}, "namespace": "ai-tenant-e2e-mt-edd1beb3", "name": "odh-trusted-ca-bundle", "reconcileID": "9a579ed5-4b27-43d4-931c-aa58e1dc670b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-edd1beb3", "update": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:43:25Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-edd1beb3"}, "namespace": "ai-tenant-e2e-mt-edd1beb3", "name": "odh-trusted-ca-bundle", "reconcileID": "9a579ed5-4b27-43d4-931c-aa58e1dc670b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-edd1beb3"}
2026-07-08T17:43:25Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-edd1beb3"}, "namespace": "ai-tenant-e2e-mt-edd1beb3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "59bac379-3084-4fb2-b59a-be569b157d21", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-edd1beb3"}
2026-07-08T17:43:26Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-5b6b4a28"}, "namespace": "ai-tenant-e2e-mt-5b6b4a28", "name": "openshift-service-ca.crt", "reconcileID": "ff7d8858-b844-4760-91dd-87ebdff9d370", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-5b6b4a28"}
2026-07-08T17:43:26Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-5b6b4a28"}, "namespace": "ai-tenant-e2e-mt-5b6b4a28", "name": "odh-trusted-ca-bundle", "reconcileID": "d689aa11-db08-4c36-b0b4-e8fcceb5a7f8", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-5b6b4a28"}
2026-07-08T17:43:26Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-5b6b4a28"}, "namespace": "ai-tenant-e2e-mt-5b6b4a28", "name": "openshift-service-ca.crt", "reconcileID": "dc645fc6-fd35-4a88-82cd-71ae6313f4a4", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-5b6b4a28", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:43:26Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-5b6b4a28"}, "namespace": "ai-tenant-e2e-mt-5b6b4a28", "name": "openshift-service-ca.crt", "reconcileID": "dc645fc6-fd35-4a88-82cd-71ae6313f4a4", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-5b6b4a28"}
2026-07-08T17:43:26Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-5b6b4a28"}, "namespace": "ai-tenant-e2e-mt-5b6b4a28", "name": "openshift-service-ca.crt", "reconcileID": "e4f6c6dd-e2b0-4852-80f2-970957287331", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-5b6b4a28"}
2026-07-08T17:43:26Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-5b6b4a28"}, "namespace": "ai-tenant-e2e-mt-5b6b4a28", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "11d4653f-7e18-442c-8987-fb33f3bde26e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-5b6b4a28"}
2026-07-08T17:43:26Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-5b6b4a28"}, "namespace": "ai-tenant-e2e-mt-5b6b4a28", "name": "odh-trusted-ca-bundle", "reconcileID": "eec01fc8-c460-4526-9237-5d2ec0310c50", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-5b6b4a28", "update": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:43:26Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-5b6b4a28"}, "namespace": "ai-tenant-e2e-mt-5b6b4a28", "name": "odh-trusted-ca-bundle", "reconcileID": "eec01fc8-c460-4526-9237-5d2ec0310c50", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-5b6b4a28"}
2026-07-08T17:43:26Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-5b6b4a28"}, "namespace": "ai-tenant-e2e-mt-5b6b4a28", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ee35dd06-fe2d-4662-bec0-f43e90531bd1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-5b6b4a28"}
2026-07-08T17:43:33Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-edd1beb3"}, "namespace": "ai-tenant-e2e-mt-edd1beb3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bbcd439b-6c01-4b5f-b1ac-1dc246962f28", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-edd1beb3"}
2026-07-08T17:43:33Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-edd1beb3"}, "namespace": "ai-tenant-e2e-mt-edd1beb3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bbcd439b-6c01-4b5f-b1ac-1dc246962f28", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-edd1beb3", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:43:33Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-edd1beb3"}, "namespace": "ai-tenant-e2e-mt-edd1beb3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bbcd439b-6c01-4b5f-b1ac-1dc246962f28", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-edd1beb3 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-08T17:43:33Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-edd1beb3"}, "namespace": "ai-tenant-e2e-mt-edd1beb3", "name": "odh-trusted-ca-bundle", "reconcileID": "a0f41a77-1b02-46ad-99c8-6fdd679cd881", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-edd1beb3"}
2026-07-08T17:43:33Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-edd1beb3"}, "namespace": "ai-tenant-e2e-mt-edd1beb3", "name": "odh-trusted-ca-bundle", "reconcileID": "a0f41a77-1b02-46ad-99c8-6fdd679cd881", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-edd1beb3"}
2026-07-08T17:43:33Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-edd1beb3"}, "namespace": "ai-tenant-e2e-mt-edd1beb3", "name": "openshift-service-ca.crt", "reconcileID": "230721af-3081-4bc8-9ebe-f2db1f2df3af", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-edd1beb3"}
2026-07-08T17:43:33Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-edd1beb3"}, "namespace": "ai-tenant-e2e-mt-edd1beb3", "name": "openshift-service-ca.crt", "reconcileID": "230721af-3081-4bc8-9ebe-f2db1f2df3af", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-edd1beb3"}
2026-07-08T17:43:33Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-edd1beb3"}, "namespace": "ai-tenant-e2e-mt-edd1beb3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "23093958-6beb-4506-868b-d7f216e9866c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-edd1beb3"}
2026-07-08T17:43:33Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-edd1beb3"}, "namespace": "ai-tenant-e2e-mt-edd1beb3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "23093958-6beb-4506-868b-d7f216e9866c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-edd1beb3"}
2026-07-08T17:43:39Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-5b6b4a28"}, "namespace": "ai-tenant-e2e-mt-5b6b4a28", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8541c011-9ddd-48b1-b27e-b8bef4fdf63c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-5b6b4a28"}
2026-07-08T17:43:39Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-5b6b4a28"}, "namespace": "ai-tenant-e2e-mt-5b6b4a28", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8541c011-9ddd-48b1-b27e-b8bef4fdf63c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-5b6b4a28", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:43:39Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-5b6b4a28"}, "namespace": "ai-tenant-e2e-mt-5b6b4a28", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8541c011-9ddd-48b1-b27e-b8bef4fdf63c", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-5b6b4a28 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-08T17:43:39Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-5b6b4a28"}, "namespace": "ai-tenant-e2e-mt-5b6b4a28", "name": "odh-trusted-ca-bundle", "reconcileID": "394f3678-8f4b-418a-afb6-2f841769ef7d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-5b6b4a28"}
2026-07-08T17:43:39Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-5b6b4a28"}, "namespace": "ai-tenant-e2e-mt-5b6b4a28", "name": "odh-trusted-ca-bundle", "reconcileID": "394f3678-8f4b-418a-afb6-2f841769ef7d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-5b6b4a28"}
2026-07-08T17:43:39Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-5b6b4a28"}, "namespace": "ai-tenant-e2e-mt-5b6b4a28", "name": "openshift-service-ca.crt", "reconcileID": "9a7bd295-4285-4b40-bcc2-417073986a19", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-5b6b4a28"}
2026-07-08T17:43:39Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-5b6b4a28"}, "namespace": "ai-tenant-e2e-mt-5b6b4a28", "name": "openshift-service-ca.crt", "reconcileID": "9a7bd295-4285-4b40-bcc2-417073986a19", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-5b6b4a28"}
2026-07-08T17:43:39Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-5b6b4a28"}, "namespace": "ai-tenant-e2e-mt-5b6b4a28", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "46247d25-4114-4442-882f-11152ebbd73b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-5b6b4a28"}
2026-07-08T17:43:39Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-5b6b4a28"}, "namespace": "ai-tenant-e2e-mt-5b6b4a28", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "46247d25-4114-4442-882f-11152ebbd73b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-5b6b4a28"}
2026-07-08T17:43:40Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d52bb8d4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4", "reconcileID": "08f46513-705f-4fc4-aa26-af18706812fd", "Gateway": "e2e-mt-d52bb8d4", "namespace": "openshift-ingress"}
2026-07-08T17:43:40Z	INFO	Creating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d52bb8d4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4", "reconcileID": "08f46513-705f-4fc4-aa26-af18706812fd", "Gateway": "e2e-mt-d52bb8d4", "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4-authn-ssl"}
2026-07-08T17:43:40Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d52bb8d4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4", "reconcileID": "08f46513-705f-4fc4-aa26-af18706812fd", "Gateway": "e2e-mt-d52bb8d4", "namespace": "openshift-ingress"}
2026-07-08T17:43:40Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d52bb8d4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4", "reconcileID": "850fcf99-1248-4cba-a33d-c7faf18e94a2", "Gateway": "e2e-mt-d52bb8d4", "namespace": "openshift-ingress"}
2026-07-08T17:43:40Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d52bb8d4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4", "reconcileID": "850fcf99-1248-4cba-a33d-c7faf18e94a2", "Gateway": "e2e-mt-d52bb8d4", "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4-authn-ssl"}
2026-07-08T17:43:40Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d52bb8d4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4", "reconcileID": "850fcf99-1248-4cba-a33d-c7faf18e94a2", "Gateway": "e2e-mt-d52bb8d4", "namespace": "openshift-ingress"}
2026-07-08T17:43:40Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d52bb8d4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4", "reconcileID": "be9c949a-d983-412c-87b9-592bfbdf1b54", "Gateway": "e2e-mt-d52bb8d4", "namespace": "openshift-ingress"}
2026-07-08T17:43:40Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d52bb8d4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4", "reconcileID": "be9c949a-d983-412c-87b9-592bfbdf1b54", "Gateway": "e2e-mt-d52bb8d4", "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4-authn-ssl"}
2026-07-08T17:43:40Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d52bb8d4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4", "reconcileID": "be9c949a-d983-412c-87b9-592bfbdf1b54", "Gateway": "e2e-mt-d52bb8d4", "namespace": "openshift-ingress"}
2026-07-08T17:43:40Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d52bb8d4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4", "reconcileID": "09d522cb-2573-4fc6-9de1-566a61f96937", "Gateway": "e2e-mt-d52bb8d4", "namespace": "openshift-ingress"}
2026-07-08T17:43:40Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d52bb8d4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4", "reconcileID": "09d522cb-2573-4fc6-9de1-566a61f96937", "Gateway": "e2e-mt-d52bb8d4", "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4-authn-ssl"}
2026-07-08T17:43:40Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d52bb8d4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4", "reconcileID": "09d522cb-2573-4fc6-9de1-566a61f96937", "Gateway": "e2e-mt-d52bb8d4", "namespace": "openshift-ingress"}
2026-07-08T17:43:41Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d52bb8d4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4", "reconcileID": "7f2a1280-640d-45bb-8342-154703fbebb4", "Gateway": "e2e-mt-d52bb8d4", "namespace": "openshift-ingress"}
2026-07-08T17:43:41Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d52bb8d4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4", "reconcileID": "7f2a1280-640d-45bb-8342-154703fbebb4", "Gateway": "e2e-mt-d52bb8d4", "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4-authn-ssl"}
2026-07-08T17:43:41Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d52bb8d4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4", "reconcileID": "7f2a1280-640d-45bb-8342-154703fbebb4", "Gateway": "e2e-mt-d52bb8d4", "namespace": "openshift-ingress"}
2026-07-08T17:43:41Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d52bb8d4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4", "reconcileID": "b261df27-6688-4d35-a2e1-5d3c07db8368", "Gateway": "e2e-mt-d52bb8d4", "namespace": "openshift-ingress"}
2026-07-08T17:43:41Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d52bb8d4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4", "reconcileID": "b261df27-6688-4d35-a2e1-5d3c07db8368", "Gateway": "e2e-mt-d52bb8d4", "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4-authn-ssl"}
2026-07-08T17:43:41Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d52bb8d4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4", "reconcileID": "b261df27-6688-4d35-a2e1-5d3c07db8368", "Gateway": "e2e-mt-d52bb8d4", "namespace": "openshift-ingress"}
2026-07-08T17:43:41Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d52bb8d4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4", "reconcileID": "66ff5f60-d774-4e0b-8093-90a62607e81f", "Gateway": "e2e-mt-d52bb8d4", "namespace": "openshift-ingress"}
2026-07-08T17:43:41Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d52bb8d4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4", "reconcileID": "66ff5f60-d774-4e0b-8093-90a62607e81f", "Gateway": "e2e-mt-d52bb8d4", "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4-authn-ssl"}
2026-07-08T17:43:41Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d52bb8d4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4", "reconcileID": "66ff5f60-d774-4e0b-8093-90a62607e81f", "Gateway": "e2e-mt-d52bb8d4", "namespace": "openshift-ingress"}
2026-07-08T17:46:45Z	DEBUG	Gateway not found, resources owned by Gateway will be garbage collected	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d52bb8d4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d52bb8d4", "reconcileID": "8cb783c4-3dc3-4717-8502-273ca3483437", "Gateway": "e2e-mt-d52bb8d4", "namespace": "openshift-ingress"}
2026-07-08T17:46:46Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-fc93d604"}, "namespace": "e2e-webhook-sub-fc93d604", "name": "openshift-service-ca.crt", "reconcileID": "1b4ec89d-5ad1-44fa-85a7-a377127bcd57", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-fc93d604"}
2026-07-08T17:46:46Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-fc93d604"}, "namespace": "e2e-webhook-sub-fc93d604", "name": "odh-trusted-ca-bundle", "reconcileID": "a0d8df33-f918-4daa-86ec-d81fa0d46db7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-fc93d604", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:46:46Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-fc93d604"}, "namespace": "e2e-webhook-sub-fc93d604", "name": "odh-trusted-ca-bundle", "reconcileID": "a0d8df33-f918-4daa-86ec-d81fa0d46db7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-fc93d604"}
2026-07-08T17:46:46Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-fc93d604"}, "namespace": "e2e-webhook-sub-fc93d604", "name": "openshift-service-ca.crt", "reconcileID": "c5fc25af-dfc4-44c9-9cf8-bb4c7d89974f", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-fc93d604", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:46:46Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-fc93d604"}, "namespace": "e2e-webhook-sub-fc93d604", "name": "openshift-service-ca.crt", "reconcileID": "c5fc25af-dfc4-44c9-9cf8-bb4c7d89974f", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-08T17:46:46Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-fc93d604"}, "namespace": "e2e-webhook-sub-fc93d604", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "55d6d4b9-09f5-4c5f-a168-f667de45d133", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-fc93d604", "update": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:46:46Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-fc93d604"}, "namespace": "e2e-webhook-sub-fc93d604", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "55d6d4b9-09f5-4c5f-a168-f667de45d133", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-fc93d604"}
2026-07-08T17:46:46Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-fc93d604"}, "namespace": "e2e-webhook-sub-fc93d604", "name": "odh-trusted-ca-bundle", "reconcileID": "0bbbf0e9-cbfb-4785-b700-9d23cd66dcba", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-fc93d604"}
2026-07-08T17:46:46Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-fc93d604"}, "namespace": "e2e-webhook-sub-fc93d604", "name": "openshift-service-ca.crt", "reconcileID": "13cb3a19-4a66-44fe-8a00-1c6654becab7", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-fc93d604"}
2026-07-08T17:46:46Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-fc93d604"}, "namespace": "e2e-webhook-sub-fc93d604", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "00fee055-6a5a-44b4-b7a3-8587e97a1b9f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-fc93d604"}
2026-07-08T17:46:52Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-fc93d604"}, "namespace": "e2e-webhook-sub-fc93d604", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f77dac4e-af0d-4a20-9249-9a84eb890bf7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-fc93d604"}
2026-07-08T17:46:52Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-fc93d604"}, "namespace": "e2e-webhook-sub-fc93d604", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f77dac4e-af0d-4a20-9249-9a84eb890bf7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-fc93d604", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:46:52Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-fc93d604"}, "namespace": "e2e-webhook-sub-fc93d604", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f77dac4e-af0d-4a20-9249-9a84eb890bf7", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-sub-fc93d604 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-08T17:46:52Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-fc93d604"}, "namespace": "e2e-webhook-sub-fc93d604", "name": "odh-trusted-ca-bundle", "reconcileID": "8ad32e81-0fd0-4d3c-9237-50bfbac312f0", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-fc93d604"}
2026-07-08T17:46:52Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-fc93d604"}, "namespace": "e2e-webhook-sub-fc93d604", "name": "odh-trusted-ca-bundle", "reconcileID": "8ad32e81-0fd0-4d3c-9237-50bfbac312f0", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-fc93d604"}
2026-07-08T17:46:52Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-fc93d604"}, "namespace": "e2e-webhook-sub-fc93d604", "name": "openshift-service-ca.crt", "reconcileID": "91455604-cf68-4aa6-8b4b-a58d9fa8c4fd", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-fc93d604"}
2026-07-08T17:46:52Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-fc93d604"}, "namespace": "e2e-webhook-sub-fc93d604", "name": "openshift-service-ca.crt", "reconcileID": "91455604-cf68-4aa6-8b4b-a58d9fa8c4fd", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-fc93d604"}
2026-07-08T17:46:52Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-fc93d604"}, "namespace": "e2e-webhook-sub-fc93d604", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "be3f6969-dda3-4cde-bb95-3cb55070998c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-fc93d604"}
2026-07-08T17:46:52Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-fc93d604"}, "namespace": "e2e-webhook-sub-fc93d604", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "be3f6969-dda3-4cde-bb95-3cb55070998c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-fc93d604"}
2026-07-08T17:46:52Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-d35fe7ec"}, "namespace": "e2e-webhook-auth-d35fe7ec", "name": "openshift-service-ca.crt", "reconcileID": "76c7ac71-2b69-46e9-a383-84fa6b2a60aa", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-d35fe7ec"}
2026-07-08T17:46:52Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-d35fe7ec"}, "namespace": "e2e-webhook-auth-d35fe7ec", "name": "odh-trusted-ca-bundle", "reconcileID": "f33c9630-24e3-4531-aa70-f3068efd26e0", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-d35fe7ec"}
2026-07-08T17:46:52Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-d35fe7ec"}, "namespace": "e2e-webhook-auth-d35fe7ec", "name": "openshift-service-ca.crt", "reconcileID": "2a2b83ba-1440-40fd-95d2-3b5ed6968e91", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-d35fe7ec", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:46:52Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-d35fe7ec"}, "namespace": "e2e-webhook-auth-d35fe7ec", "name": "openshift-service-ca.crt", "reconcileID": "2a2b83ba-1440-40fd-95d2-3b5ed6968e91", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-d35fe7ec"}
2026-07-08T17:46:52Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-d35fe7ec"}, "namespace": "e2e-webhook-auth-d35fe7ec", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "09fd8c9b-c3bc-4922-a532-6b27d158b243", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-d35fe7ec"}
2026-07-08T17:46:52Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-d35fe7ec"}, "namespace": "e2e-webhook-auth-d35fe7ec", "name": "openshift-service-ca.crt", "reconcileID": "1aed64e8-acc7-4b7c-86d6-1cb58cf65a64", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-d35fe7ec"}
2026-07-08T17:46:52Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-d35fe7ec"}, "namespace": "e2e-webhook-auth-d35fe7ec", "name": "odh-trusted-ca-bundle", "reconcileID": "ae96c28b-0cfa-484a-b5b8-40f9cd07b785", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-d35fe7ec", "update": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:46:52Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-d35fe7ec"}, "namespace": "e2e-webhook-auth-d35fe7ec", "name": "odh-trusted-ca-bundle", "reconcileID": "ae96c28b-0cfa-484a-b5b8-40f9cd07b785", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-d35fe7ec"}
2026-07-08T17:46:52Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-d35fe7ec"}, "namespace": "e2e-webhook-auth-d35fe7ec", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d6b841b6-8ea9-47f6-9f36-3c1c330d179d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-d35fe7ec"}
2026-07-08T17:46:58Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-d35fe7ec"}, "namespace": "e2e-webhook-auth-d35fe7ec", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "22693045-dc4b-4294-9e60-b7bc74232988", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-d35fe7ec"}
2026-07-08T17:46:58Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-d35fe7ec"}, "namespace": "e2e-webhook-auth-d35fe7ec", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "22693045-dc4b-4294-9e60-b7bc74232988", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-d35fe7ec", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:46:58Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-d35fe7ec"}, "namespace": "e2e-webhook-auth-d35fe7ec", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "22693045-dc4b-4294-9e60-b7bc74232988", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-auth-d35fe7ec because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-08T17:46:58Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-d35fe7ec"}, "namespace": "e2e-webhook-auth-d35fe7ec", "name": "odh-trusted-ca-bundle", "reconcileID": "1bd7e63c-743d-470b-be4d-5e6fb5ec8112", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-d35fe7ec"}
2026-07-08T17:46:58Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-d35fe7ec"}, "namespace": "e2e-webhook-auth-d35fe7ec", "name": "odh-trusted-ca-bundle", "reconcileID": "1bd7e63c-743d-470b-be4d-5e6fb5ec8112", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-d35fe7ec"}
2026-07-08T17:46:58Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-d35fe7ec"}, "namespace": "e2e-webhook-auth-d35fe7ec", "name": "openshift-service-ca.crt", "reconcileID": "198e18a8-8355-40cd-8b7b-1eb8e484e7b7", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-d35fe7ec"}
2026-07-08T17:46:58Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-d35fe7ec"}, "namespace": "e2e-webhook-auth-d35fe7ec", "name": "openshift-service-ca.crt", "reconcileID": "198e18a8-8355-40cd-8b7b-1eb8e484e7b7", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-d35fe7ec"}
2026-07-08T17:46:58Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-d35fe7ec"}, "namespace": "e2e-webhook-auth-d35fe7ec", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7fdfba10-c7ef-47f6-a79b-e10b87a5a19a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-d35fe7ec"}
2026-07-08T17:46:58Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-d35fe7ec"}, "namespace": "e2e-webhook-auth-d35fe7ec", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7fdfba10-c7ef-47f6-a79b-e10b87a5a19a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-d35fe7ec"}
2026-07-08T17:47:05Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7efd5629","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629", "reconcileID": "ca2d654f-ff53-4519-9bb7-051a691878fe", "Gateway": "e2e-mt-7efd5629", "namespace": "openshift-ingress"}
2026-07-08T17:47:05Z	INFO	Creating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7efd5629","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629", "reconcileID": "ca2d654f-ff53-4519-9bb7-051a691878fe", "Gateway": "e2e-mt-7efd5629", "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629-authn-ssl"}
2026-07-08T17:47:05Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7efd5629","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629", "reconcileID": "ca2d654f-ff53-4519-9bb7-051a691878fe", "Gateway": "e2e-mt-7efd5629", "namespace": "openshift-ingress"}
2026-07-08T17:47:05Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7efd5629","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629", "reconcileID": "7c181824-187e-43f2-a2b3-ee6e38ce55c0", "Gateway": "e2e-mt-7efd5629", "namespace": "openshift-ingress"}
2026-07-08T17:47:05Z	INFO	Creating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7efd5629","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629", "reconcileID": "7c181824-187e-43f2-a2b3-ee6e38ce55c0", "Gateway": "e2e-mt-7efd5629", "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629-authn-ssl"}
2026-07-08T17:47:05Z	ERROR	Reconciler error	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7efd5629","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629", "reconcileID": "7c181824-187e-43f2-a2b3-ee6e38ce55c0", "error": "failed to create EnvoyFilter: could not CREATE EnvoyFilter openshift-ingress/e2e-mt-7efd5629-authn-ssl: envoyfilters.networking.istio.io \"e2e-mt-7efd5629-authn-ssl\" already exists"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-08T17:47:05Z	DEBUG	events	Failed to reconcile EnvoyFilter: failed to create EnvoyFilter: could not CREATE EnvoyFilter openshift-ingress/e2e-mt-7efd5629-authn-ssl: envoyfilters.networking.istio.io "e2e-mt-7efd5629-authn-ssl" already exists	{"type": "Warning", "object": {"kind":"Gateway","namespace":"openshift-ingress","name":"e2e-mt-7efd5629","uid":"d3c55bcd-25d4-443f-951f-4ab5d25fd8b2","apiVersion":"gateway.networking.k8s.io/v1","resourceVersion":"48026"}, "reason": "ReconcileError"}
2026-07-08T17:47:05Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7efd5629","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629", "reconcileID": "eab5e9f0-1314-4e92-b033-1df641831c72", "Gateway": "e2e-mt-7efd5629", "namespace": "openshift-ingress"}
2026-07-08T17:47:05Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7efd5629","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629", "reconcileID": "eab5e9f0-1314-4e92-b033-1df641831c72", "Gateway": "e2e-mt-7efd5629", "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629-authn-ssl"}
2026-07-08T17:47:05Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7efd5629","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629", "reconcileID": "eab5e9f0-1314-4e92-b033-1df641831c72", "Gateway": "e2e-mt-7efd5629", "namespace": "openshift-ingress"}
2026-07-08T17:47:05Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7efd5629","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629", "reconcileID": "50848db5-7d99-454d-810d-0205034e9655", "Gateway": "e2e-mt-7efd5629", "namespace": "openshift-ingress"}
2026-07-08T17:47:05Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7efd5629","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629", "reconcileID": "50848db5-7d99-454d-810d-0205034e9655", "Gateway": "e2e-mt-7efd5629", "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629-authn-ssl"}
2026-07-08T17:47:05Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7efd5629","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629", "reconcileID": "50848db5-7d99-454d-810d-0205034e9655", "Gateway": "e2e-mt-7efd5629", "namespace": "openshift-ingress"}
2026-07-08T17:47:06Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7efd5629","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629", "reconcileID": "3a12357e-96fd-49a9-8f02-e33ad2008653", "Gateway": "e2e-mt-7efd5629", "namespace": "openshift-ingress"}
2026-07-08T17:47:06Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7efd5629","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629", "reconcileID": "3a12357e-96fd-49a9-8f02-e33ad2008653", "Gateway": "e2e-mt-7efd5629", "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629-authn-ssl"}
2026-07-08T17:47:06Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7efd5629","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629", "reconcileID": "3a12357e-96fd-49a9-8f02-e33ad2008653", "Gateway": "e2e-mt-7efd5629", "namespace": "openshift-ingress"}
2026-07-08T17:47:06Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7efd5629","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629", "reconcileID": "13680af8-b7b2-4d9f-aef4-9b100308cb23", "Gateway": "e2e-mt-7efd5629", "namespace": "openshift-ingress"}
2026-07-08T17:47:06Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7efd5629","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629", "reconcileID": "13680af8-b7b2-4d9f-aef4-9b100308cb23", "Gateway": "e2e-mt-7efd5629", "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629-authn-ssl"}
2026-07-08T17:47:06Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7efd5629","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629", "reconcileID": "13680af8-b7b2-4d9f-aef4-9b100308cb23", "Gateway": "e2e-mt-7efd5629", "namespace": "openshift-ingress"}
2026-07-08T17:47:06Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7efd5629","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629", "reconcileID": "27722d37-4cc9-48c5-88d4-dc846b0a794b", "Gateway": "e2e-mt-7efd5629", "namespace": "openshift-ingress"}
2026-07-08T17:47:06Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7efd5629","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629", "reconcileID": "27722d37-4cc9-48c5-88d4-dc846b0a794b", "Gateway": "e2e-mt-7efd5629", "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629-authn-ssl"}
2026-07-08T17:47:06Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7efd5629","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629", "reconcileID": "27722d37-4cc9-48c5-88d4-dc846b0a794b", "Gateway": "e2e-mt-7efd5629", "namespace": "openshift-ingress"}
2026-07-08T17:48:05Z	INFO	No monitoring namespace detected, skipping monitoring reconciliation.	{"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"}
2026-07-08T17:48:05Z	INFO	Multi Node reconciling.	{"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"}
2026-07-08T17:48:07Z	INFO	ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt	{"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"}
2026-07-08T17:48:07Z	INFO	Multi Node reconciled successfully.	{"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"}
2026-07-08T17:50:12Z	DEBUG	Gateway not found, resources owned by Gateway will be garbage collected	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-7efd5629","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-7efd5629", "reconcileID": "d991e141-04c1-47eb-b9af-150710b525bc", "Gateway": "e2e-mt-7efd5629", "namespace": "openshift-ingress"}
2026-07-08T17:50:14Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-47e0b6c6"}, "namespace": "ai-tenant-e2e-mt-47e0b6c6", "name": "openshift-service-ca.crt", "reconcileID": "b9e051a6-2218-4b17-b969-526b8a497deb", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-47e0b6c6"}
2026-07-08T17:50:14Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-47e0b6c6"}, "namespace": "ai-tenant-e2e-mt-47e0b6c6", "name": "odh-trusted-ca-bundle", "reconcileID": "7518f1ea-7de3-4864-add9-6bd86207d716", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-47e0b6c6"}
2026-07-08T17:50:14Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-47e0b6c6"}, "namespace": "ai-tenant-e2e-mt-47e0b6c6", "name": "openshift-service-ca.crt", "reconcileID": "7898ef83-9cf3-4c5b-99db-5003e7a321c3", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-47e0b6c6", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:50:14Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-47e0b6c6"}, "namespace": "ai-tenant-e2e-mt-47e0b6c6", "name": "openshift-service-ca.crt", "reconcileID": "7898ef83-9cf3-4c5b-99db-5003e7a321c3", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-47e0b6c6"}
2026-07-08T17:50:14Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-47e0b6c6"}, "namespace": "ai-tenant-e2e-mt-47e0b6c6", "name": "openshift-service-ca.crt", "reconcileID": "49224c2a-6409-4643-9e92-4464628b4b6f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-47e0b6c6"}
2026-07-08T17:50:14Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-47e0b6c6"}, "namespace": "ai-tenant-e2e-mt-47e0b6c6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0e40d24c-b67a-4323-977c-adebf845731d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-47e0b6c6"}
2026-07-08T17:50:14Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-47e0b6c6"}, "namespace": "ai-tenant-e2e-mt-47e0b6c6", "name": "odh-trusted-ca-bundle", "reconcileID": "c3854bcc-a144-441b-b2bb-c435c848740e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-47e0b6c6", "update": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:50:14Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-47e0b6c6"}, "namespace": "ai-tenant-e2e-mt-47e0b6c6", "name": "odh-trusted-ca-bundle", "reconcileID": "c3854bcc-a144-441b-b2bb-c435c848740e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-47e0b6c6"}
2026-07-08T17:50:14Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-47e0b6c6"}, "namespace": "ai-tenant-e2e-mt-47e0b6c6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d2f5da8a-dbce-416d-b0c5-cd54923df515", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-47e0b6c6"}
2026-07-08T17:50:15Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-efa5e17f"}, "namespace": "ai-tenant-e2e-mt-efa5e17f", "name": "openshift-service-ca.crt", "reconcileID": "38dd665b-091a-4149-9166-687ce5dd0fff", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-efa5e17f"}
2026-07-08T17:50:15Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-efa5e17f"}, "namespace": "ai-tenant-e2e-mt-efa5e17f", "name": "odh-trusted-ca-bundle", "reconcileID": "69a420fe-729b-4c3a-a7ee-bf3bf610c17f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-efa5e17f"}
2026-07-08T17:50:15Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-efa5e17f"}, "namespace": "ai-tenant-e2e-mt-efa5e17f", "name": "openshift-service-ca.crt", "reconcileID": "a61f5755-5a57-4793-8bee-10ef3a12172d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-efa5e17f", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:50:15Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-efa5e17f"}, "namespace": "ai-tenant-e2e-mt-efa5e17f", "name": "openshift-service-ca.crt", "reconcileID": "a61f5755-5a57-4793-8bee-10ef3a12172d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-efa5e17f"}
2026-07-08T17:50:15Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-efa5e17f"}, "namespace": "ai-tenant-e2e-mt-efa5e17f", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "29566358-486e-42fe-8e0a-c4a8a90bb0bb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-efa5e17f"}
2026-07-08T17:50:15Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-efa5e17f"}, "namespace": "ai-tenant-e2e-mt-efa5e17f", "name": "openshift-service-ca.crt", "reconcileID": "a262ec61-ae12-4026-9e77-e7df14532d9c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-efa5e17f"}
2026-07-08T17:50:15Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-efa5e17f"}, "namespace": "ai-tenant-e2e-mt-efa5e17f", "name": "odh-trusted-ca-bundle", "reconcileID": "0e5e1052-d404-4b11-9da8-8df2b2d99293", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-efa5e17f", "update": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:50:15Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-efa5e17f"}, "namespace": "ai-tenant-e2e-mt-efa5e17f", "name": "odh-trusted-ca-bundle", "reconcileID": "0e5e1052-d404-4b11-9da8-8df2b2d99293", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-efa5e17f"}
2026-07-08T17:50:15Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-efa5e17f"}, "namespace": "ai-tenant-e2e-mt-efa5e17f", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b9d8b49f-3a30-4419-bc2d-ee5ab4ed7c68", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-efa5e17f"}
2026-07-08T17:50:22Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-47e0b6c6"}, "namespace": "ai-tenant-e2e-mt-47e0b6c6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4e3efcab-4186-4a5a-b885-88b7820e08e2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-47e0b6c6"}
2026-07-08T17:50:22Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-47e0b6c6"}, "namespace": "ai-tenant-e2e-mt-47e0b6c6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4e3efcab-4186-4a5a-b885-88b7820e08e2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-47e0b6c6", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:50:22Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-47e0b6c6"}, "namespace": "ai-tenant-e2e-mt-47e0b6c6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4e3efcab-4186-4a5a-b885-88b7820e08e2", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-47e0b6c6 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-08T17:50:22Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-47e0b6c6"}, "namespace": "ai-tenant-e2e-mt-47e0b6c6", "name": "odh-trusted-ca-bundle", "reconcileID": "1a3e02a3-29dc-495e-ab83-606e360440c3", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-47e0b6c6"}
2026-07-08T17:50:22Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-47e0b6c6"}, "namespace": "ai-tenant-e2e-mt-47e0b6c6", "name": "odh-trusted-ca-bundle", "reconcileID": "1a3e02a3-29dc-495e-ab83-606e360440c3", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-47e0b6c6"}
2026-07-08T17:50:22Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-47e0b6c6"}, "namespace": "ai-tenant-e2e-mt-47e0b6c6", "name": "openshift-service-ca.crt", "reconcileID": "75b0a34c-135b-45d9-ac00-330831ae6dc1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-47e0b6c6"}
2026-07-08T17:50:22Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-47e0b6c6"}, "namespace": "ai-tenant-e2e-mt-47e0b6c6", "name": "openshift-service-ca.crt", "reconcileID": "75b0a34c-135b-45d9-ac00-330831ae6dc1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-47e0b6c6"}
2026-07-08T17:50:22Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-47e0b6c6"}, "namespace": "ai-tenant-e2e-mt-47e0b6c6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0406e437-12bf-43cf-b702-fe2ade55b7d8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-47e0b6c6"}
2026-07-08T17:50:22Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-47e0b6c6"}, "namespace": "ai-tenant-e2e-mt-47e0b6c6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0406e437-12bf-43cf-b702-fe2ade55b7d8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-47e0b6c6"}
2026-07-08T17:50:29Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-efa5e17f"}, "namespace": "ai-tenant-e2e-mt-efa5e17f", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "57365a6f-b3ad-4c9c-bf3c-96b74e857770", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-efa5e17f"}
2026-07-08T17:50:29Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-efa5e17f"}, "namespace": "ai-tenant-e2e-mt-efa5e17f", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "57365a6f-b3ad-4c9c-bf3c-96b74e857770", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-efa5e17f", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:50:29Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-efa5e17f"}, "namespace": "ai-tenant-e2e-mt-efa5e17f", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "57365a6f-b3ad-4c9c-bf3c-96b74e857770", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-efa5e17f because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-08T17:50:29Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-efa5e17f"}, "namespace": "ai-tenant-e2e-mt-efa5e17f", "name": "odh-trusted-ca-bundle", "reconcileID": "ffe97a3a-a4aa-4cf6-bafe-fc9812f9b060", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-efa5e17f"}
2026-07-08T17:50:29Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-efa5e17f"}, "namespace": "ai-tenant-e2e-mt-efa5e17f", "name": "odh-trusted-ca-bundle", "reconcileID": "ffe97a3a-a4aa-4cf6-bafe-fc9812f9b060", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-efa5e17f"}
2026-07-08T17:50:29Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-efa5e17f"}, "namespace": "ai-tenant-e2e-mt-efa5e17f", "name": "openshift-service-ca.crt", "reconcileID": "1cda4498-6590-4c75-824b-9d411873099e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-efa5e17f"}
2026-07-08T17:50:29Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-efa5e17f"}, "namespace": "ai-tenant-e2e-mt-efa5e17f", "name": "openshift-service-ca.crt", "reconcileID": "1cda4498-6590-4c75-824b-9d411873099e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-efa5e17f"}
2026-07-08T17:50:29Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-efa5e17f"}, "namespace": "ai-tenant-e2e-mt-efa5e17f", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "81fba4ca-4059-4f8c-b3ea-2500fa613151", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-efa5e17f"}
2026-07-08T17:50:29Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-efa5e17f"}, "namespace": "ai-tenant-e2e-mt-efa5e17f", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "81fba4ca-4059-4f8c-b3ea-2500fa613151", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-efa5e17f"}
2026-07-08T17:50:29Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-eb8d9505"}, "namespace": "ai-tenant-e2e-mt-eb8d9505", "name": "openshift-service-ca.crt", "reconcileID": "27375978-aed4-4c0e-8ddd-7a25b9fa6f5c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-eb8d9505"}
2026-07-08T17:50:29Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-eb8d9505"}, "namespace": "ai-tenant-e2e-mt-eb8d9505", "name": "odh-trusted-ca-bundle", "reconcileID": "e12df801-4d15-4b9c-9e9d-bbb6a6e9aa49", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-eb8d9505"}
2026-07-08T17:50:29Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-eb8d9505"}, "namespace": "ai-tenant-e2e-mt-eb8d9505", "name": "openshift-service-ca.crt", "reconcileID": "a84dee0b-281a-4251-9238-2c064a25edf8", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-eb8d9505", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:50:29Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-eb8d9505"}, "namespace": "ai-tenant-e2e-mt-eb8d9505", "name": "openshift-service-ca.crt", "reconcileID": "a84dee0b-281a-4251-9238-2c064a25edf8", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-eb8d9505"}
2026-07-08T17:50:29Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-eb8d9505"}, "namespace": "ai-tenant-e2e-mt-eb8d9505", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fbff07ba-5c8d-4bf0-90a5-6a86c9fbf565", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-eb8d9505"}
2026-07-08T17:50:29Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-eb8d9505"}, "namespace": "ai-tenant-e2e-mt-eb8d9505", "name": "openshift-service-ca.crt", "reconcileID": "b1f70a3a-d105-4747-960d-ad953955360d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-eb8d9505"}
2026-07-08T17:50:29Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-eb8d9505"}, "namespace": "ai-tenant-e2e-mt-eb8d9505", "name": "odh-trusted-ca-bundle", "reconcileID": "2fff5a52-f7cf-4de5-ba0e-f769f83053eb", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-eb8d9505", "update": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:50:29Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-eb8d9505"}, "namespace": "ai-tenant-e2e-mt-eb8d9505", "name": "odh-trusted-ca-bundle", "reconcileID": "2fff5a52-f7cf-4de5-ba0e-f769f83053eb", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-eb8d9505"}
2026-07-08T17:50:29Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-eb8d9505"}, "namespace": "ai-tenant-e2e-mt-eb8d9505", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bf21b847-cff6-45e1-a3ce-c5cc85124ea3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-eb8d9505"}
2026-07-08T17:51:06Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-eb8d9505"}, "namespace": "ai-tenant-e2e-mt-eb8d9505", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dac33131-0153-4a26-bfc0-18b1e9cc031e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-eb8d9505"}
2026-07-08T17:51:06Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-eb8d9505"}, "namespace": "ai-tenant-e2e-mt-eb8d9505", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dac33131-0153-4a26-bfc0-18b1e9cc031e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-eb8d9505", "create": "odh-kserve-custom-ca-bundle"}
2026-07-08T17:51:06Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-eb8d9505"}, "namespace": "ai-tenant-e2e-mt-eb8d9505", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dac33131-0153-4a26-bfc0-18b1e9cc031e", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-eb8d9505 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-08T17:51:06Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-eb8d9505"}, "namespace": "ai-tenant-e2e-mt-eb8d9505", "name": "odh-trusted-ca-bundle", "reconcileID": "b4267b84-68a6-4fdf-b4e6-72cd0041f790", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-eb8d9505"}
2026-07-08T17:51:06Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-eb8d9505"}, "namespace": "ai-tenant-e2e-mt-eb8d9505", "name": "odh-trusted-ca-bundle", "reconcileID": "b4267b84-68a6-4fdf-b4e6-72cd0041f790", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-eb8d9505"}
2026-07-08T17:51:06Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-eb8d9505"}, "namespace": "ai-tenant-e2e-mt-eb8d9505", "name": "openshift-service-ca.crt", "reconcileID": "d7e47b4a-5985-41e4-b0fe-ce56fe3d7398", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-eb8d9505"}
2026-07-08T17:51:06Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-eb8d9505"}, "namespace": "ai-tenant-e2e-mt-eb8d9505", "name": "openshift-service-ca.crt", "reconcileID": "d7e47b4a-5985-41e4-b0fe-ce56fe3d7398", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-eb8d9505"}
2026-07-08T17:51:06Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-eb8d9505"}, "namespace": "ai-tenant-e2e-mt-eb8d9505", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "11d9676c-3fb9-4317-b18a-9806c6863fbb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-eb8d9505"}
2026-07-08T17:51:06Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-eb8d9505"}, "namespace": "ai-tenant-e2e-mt-eb8d9505", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "11d9676c-3fb9-4317-b18a-9806c6863fbb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-eb8d9505"}
2026-07-08T17:51:07Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-780189","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-780189", "reconcileID": "b3902611-8771-4d3e-a312-947288a7f549", "Gateway": "e2e-inf-a-780189", "namespace": "openshift-ingress"}
2026-07-08T17:51:07Z	INFO	Creating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-780189","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-780189", "reconcileID": "b3902611-8771-4d3e-a312-947288a7f549", "Gateway": "e2e-inf-a-780189", "namespace": "openshift-ingress", "name": "e2e-inf-a-780189-authn-ssl"}
2026-07-08T17:51:07Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-780189","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-780189", "reconcileID": "b3902611-8771-4d3e-a312-947288a7f549", "Gateway": "e2e-inf-a-780189", "namespace": "openshift-ingress"}
2026-07-08T17:51:07Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-780189","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-780189", "reconcileID": "132c23fd-5fe4-4895-8f3b-403c3e68f6d3", "Gateway": "e2e-inf-a-780189", "namespace": "openshift-ingress"}
2026-07-08T17:51:07Z	INFO	Creating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-780189","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-780189", "reconcileID": "132c23fd-5fe4-4895-8f3b-403c3e68f6d3", "Gateway": "e2e-inf-a-780189", "namespace": "openshift-ingress", "name": "e2e-inf-a-780189-authn-ssl"}
2026-07-08T17:51:07Z	ERROR	Reconciler error	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-780189","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-780189", "reconcileID": "132c23fd-5fe4-4895-8f3b-403c3e68f6d3", "error": "failed to create EnvoyFilter: could not CREATE EnvoyFilter openshift-ingress/e2e-inf-a-780189-authn-ssl: envoyfilters.networking.istio.io \"e2e-inf-a-780189-authn-ssl\" already exists"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-08T17:51:07Z	DEBUG	events	Failed to reconcile EnvoyFilter: failed to create EnvoyFilter: could not CREATE EnvoyFilter openshift-ingress/e2e-inf-a-780189-authn-ssl: envoyfilters.networking.istio.io "e2e-inf-a-780189-authn-ssl" already exists	{"type": "Warning", "object": {"kind":"Gateway","namespace":"openshift-ingress","name":"e2e-inf-a-780189","uid":"98417010-4032-4e5f-bb8d-07587c06464f","apiVersion":"gateway.networking.k8s.io/v1","resourceVersion":"50978"}, "reason": "ReconcileError"}
2026-07-08T17:51:07Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-780189","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-780189", "reconcileID": "fd838ce9-1654-42d1-9cff-dce2c3315f8c", "Gateway": "e2e-inf-a-780189", "namespace": "openshift-ingress"}
2026-07-08T17:51:07Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-780189","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-780189", "reconcileID": "fd838ce9-1654-42d1-9cff-dce2c3315f8c", "Gateway": "e2e-inf-a-780189", "namespace": "openshift-ingress", "name": "e2e-inf-a-780189-authn-ssl"}
2026-07-08T17:51:07Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-780189","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-780189", "reconcileID": "fd838ce9-1654-42d1-9cff-dce2c3315f8c", "Gateway": "e2e-inf-a-780189", "namespace": "openshift-ingress"}
2026-07-08T17:51:07Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-780189","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-780189", "reconcileID": "96e3ab0d-75d0-45a8-ab1f-d0d5980896db", "Gateway": "e2e-inf-a-780189", "namespace": "openshift-ingress"}
2026-07-08T17:51:07Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-780189","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-780189", "reconcileID": "96e3ab0d-75d0-45a8-ab1f-d0d5980896db", "Gateway": "e2e-inf-a-780189", "namespace": "openshift-ingress", "name": "e2e-inf-a-780189-authn-ssl"}
2026-07-08T17:51:07Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-780189","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-780189", "reconcileID": "96e3ab0d-75d0-45a8-ab1f-d0d5980896db", "Gateway": "e2e-inf-a-780189", "namespace": "openshift-ingress"}
2026-07-08T17:51:08Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-780189","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-780189", "reconcileID": "5337240f-e57e-4d52-a49f-83bc32640fd8", "Gateway": "e2e-inf-a-780189", "namespace": "openshift-ingress"}
2026-07-08T17:51:08Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-780189","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-780189", "reconcileID": "5337240f-e57e-4d52-a49f-83bc32640fd8", "Gateway": "e2e-inf-a-780189", "namespace": "openshift-ingress", "name": "e2e-inf-a-780189-authn-ssl"}
2026-07-08T17:51:08Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-780189","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-780189", "reconcileID": "5337240f-e57e-4d52-a49f-83bc32640fd8", "Gateway": "e2e-inf-a-780189", "namespace": "openshift-ingress"}
2026-07-08T17:51:08Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-780189","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-780189", "reconcileID": "b6b31638-2295-47fc-a9c6-ca5dbfc8636b", "Gateway": "e2e-inf-a-780189", "namespace": "openshift-ingress"}
2026-07-08T17:51:08Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-780189","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-780189", "reconcileID": "b6b31638-2295-47fc-a9c6-ca5dbfc8636b", "Gateway": "e2e-inf-a-780189", "namespace": "openshift-ingress", "name": "e2e-inf-a-780189-authn-ssl"}
2026-07-08T17:51:08Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-780189","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-780189", "reconcileID": "b6b31638-2295-47fc-a9c6-ca5dbfc8636b", "Gateway": "e2e-inf-a-780189", "namespace": "openshift-ingress"}
2026-07-08T17:51:08Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-780189","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-780189", "reconcileID": "db453755-9008-4ad8-96b7-5bbae13ef1df", "Gateway": "e2e-inf-a-780189", "namespace": "openshift-ingress"}
2026-07-08T17:51:08Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-780189","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-780189", "reconcileID": "db453755-9008-4ad8-96b7-5bbae13ef1df", "Gateway": "e2e-inf-a-780189", "namespace": "openshift-ingress", "name": "e2e-inf-a-780189-authn-ssl"}
2026-07-08T17:51:08Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-780189","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-780189", "reconcileID": "db453755-9008-4ad8-96b7-5bbae13ef1df", "Gateway": "e2e-inf-a-780189", "namespace": "openshift-ingress"}
2026-07-08T17:54:15Z	DEBUG	Gateway not found, resources owned by Gateway will be garbage collected	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-780189","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-780189", "reconcileID": "46d9a0dc-043f-4ce0-865a-7cb5cfca55d3", "Gateway": "e2e-inf-a-780189", "namespace": "openshift-ingress"}