sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-13T03:31:13Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-derive-33013832"}, "namespace": "ai-tenant-e2e-derive-33013832", "name": "odh-trusted-ca-bundle", "reconcileID": "1c41f9c5-22d1-4874-92f3-631733e9719e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-derive-33013832"}
2026-07-13T03:31:13Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-derive-33013832"}, "namespace": "ai-tenant-e2e-derive-33013832", "name": "odh-trusted-ca-bundle", "reconcileID": "1c41f9c5-22d1-4874-92f3-631733e9719e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-derive-33013832"}
2026-07-13T03:31:13Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-derive-33013832"}, "namespace": "ai-tenant-e2e-derive-33013832", "name": "openshift-service-ca.crt", "reconcileID": "ba4bd1d5-3913-4629-91ca-87ad440fe2a3", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-derive-33013832"}
2026-07-13T03:31:13Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-derive-33013832"}, "namespace": "ai-tenant-e2e-derive-33013832", "name": "openshift-service-ca.crt", "reconcileID": "ba4bd1d5-3913-4629-91ca-87ad440fe2a3", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-derive-33013832"}
2026-07-13T03:31:13Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-derive-33013832"}, "namespace": "ai-tenant-e2e-derive-33013832", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f78cfdd7-fea7-4160-b09b-f17170264eaf", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-derive-33013832"}
2026-07-13T03:31:13Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-derive-33013832"}, "namespace": "ai-tenant-e2e-derive-33013832", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f78cfdd7-fea7-4160-b09b-f17170264eaf", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-derive-33013832"}
2026-07-13T03:31:14Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-58412af8"}, "namespace": "ai-tenant-e2e-mt-58412af8", "name": "openshift-service-ca.crt", "reconcileID": "b83714a3-4ff2-41ca-a5c0-bde28f6e188e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-58412af8"}
2026-07-13T03:31:14Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-58412af8"}, "namespace": "ai-tenant-e2e-mt-58412af8", "name": "odh-trusted-ca-bundle", "reconcileID": "28c0a8d3-ecaa-429e-b220-6b0a4058f432", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-58412af8"}
2026-07-13T03:31:14Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-58412af8"}, "namespace": "ai-tenant-e2e-mt-58412af8", "name": "openshift-service-ca.crt", "reconcileID": "04cb6223-7154-4046-861f-15fbfd48480f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-58412af8", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:31:14Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-58412af8"}, "namespace": "ai-tenant-e2e-mt-58412af8", "name": "openshift-service-ca.crt", "reconcileID": "04cb6223-7154-4046-861f-15fbfd48480f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-58412af8"}
2026-07-13T03:31:14Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-58412af8"}, "namespace": "ai-tenant-e2e-mt-58412af8", "name": "openshift-service-ca.crt", "reconcileID": "84820f9a-db4d-4619-8a79-086d6f0f54d1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-58412af8"}
2026-07-13T03:31:14Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-58412af8"}, "namespace": "ai-tenant-e2e-mt-58412af8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "52812353-723c-4988-a844-e5c0f9adf735", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-58412af8"}
2026-07-13T03:31:14Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-58412af8"}, "namespace": "ai-tenant-e2e-mt-58412af8", "name": "odh-trusted-ca-bundle", "reconcileID": "18f41316-f172-4b18-8501-b7c7b6cedc6f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-58412af8", "update": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:31:14Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-58412af8"}, "namespace": "ai-tenant-e2e-mt-58412af8", "name": "odh-trusted-ca-bundle", "reconcileID": "18f41316-f172-4b18-8501-b7c7b6cedc6f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-58412af8"}
2026-07-13T03:31:14Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-58412af8"}, "namespace": "ai-tenant-e2e-mt-58412af8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "786b12d2-605b-4971-ac84-ee36f8720cd9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-58412af8"}
2026-07-13T03:31:21Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-58412af8"}, "namespace": "ai-tenant-e2e-mt-58412af8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c19c0915-f46d-4783-b01b-8d3408678cb1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-58412af8"}
2026-07-13T03:31:21Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-58412af8"}, "namespace": "ai-tenant-e2e-mt-58412af8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c19c0915-f46d-4783-b01b-8d3408678cb1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-58412af8", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:31:21Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-58412af8"}, "namespace": "ai-tenant-e2e-mt-58412af8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c19c0915-f46d-4783-b01b-8d3408678cb1", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-58412af8 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-13T03:31:21Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-58412af8"}, "namespace": "ai-tenant-e2e-mt-58412af8", "name": "odh-trusted-ca-bundle", "reconcileID": "c5eba641-9bf4-446f-be7d-d6478c4f2fc4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-58412af8"}
2026-07-13T03:31:21Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-58412af8"}, "namespace": "ai-tenant-e2e-mt-58412af8", "name": "odh-trusted-ca-bundle", "reconcileID": "c5eba641-9bf4-446f-be7d-d6478c4f2fc4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-58412af8"}
2026-07-13T03:31:21Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-58412af8"}, "namespace": "ai-tenant-e2e-mt-58412af8", "name": "openshift-service-ca.crt", "reconcileID": "71a62c0b-096c-4806-bc12-ce54615d6720", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-58412af8"}
2026-07-13T03:31:21Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-58412af8"}, "namespace": "ai-tenant-e2e-mt-58412af8", "name": "openshift-service-ca.crt", "reconcileID": "71a62c0b-096c-4806-bc12-ce54615d6720", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-58412af8"}
2026-07-13T03:31:21Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-58412af8"}, "namespace": "ai-tenant-e2e-mt-58412af8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "71442e0c-9f5a-4c2a-998f-c7587a762a5c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-58412af8"}
2026-07-13T03:31:21Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-58412af8"}, "namespace": "ai-tenant-e2e-mt-58412af8", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "71442e0c-9f5a-4c2a-998f-c7587a762a5c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-58412af8"}
2026-07-13T03:31:21Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c120cae6"}, "namespace": "ai-tenant-e2e-mt-c120cae6", "name": "openshift-service-ca.crt", "reconcileID": "0c6786a9-7dc7-43dd-ab90-52ef41a650f3", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c120cae6"}
2026-07-13T03:31:21Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c120cae6"}, "namespace": "ai-tenant-e2e-mt-c120cae6", "name": "odh-trusted-ca-bundle", "reconcileID": "b6fe421b-82cf-4d32-a047-063a6747f7d1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c120cae6"}
2026-07-13T03:31:21Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c120cae6"}, "namespace": "ai-tenant-e2e-mt-c120cae6", "name": "openshift-service-ca.crt", "reconcileID": "37ef6f25-2eb5-4a54-b157-bfd75b210da1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c120cae6", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:31:21Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c120cae6"}, "namespace": "ai-tenant-e2e-mt-c120cae6", "name": "openshift-service-ca.crt", "reconcileID": "37ef6f25-2eb5-4a54-b157-bfd75b210da1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c120cae6"}
2026-07-13T03:31:21Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c120cae6"}, "namespace": "ai-tenant-e2e-mt-c120cae6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "db821f1e-987a-4e7e-997b-1e9eb416f67d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c120cae6"}
2026-07-13T03:31:21Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c120cae6"}, "namespace": "ai-tenant-e2e-mt-c120cae6", "name": "openshift-service-ca.crt", "reconcileID": "d19252fd-20f5-475c-af08-ab6c367ed89e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c120cae6"}
2026-07-13T03:31:21Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c120cae6"}, "namespace": "ai-tenant-e2e-mt-c120cae6", "name": "odh-trusted-ca-bundle", "reconcileID": "1882b89e-fcab-459f-b4d4-264fd2b9563f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c120cae6", "update": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:31:21Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c120cae6"}, "namespace": "ai-tenant-e2e-mt-c120cae6", "name": "odh-trusted-ca-bundle", "reconcileID": "1882b89e-fcab-459f-b4d4-264fd2b9563f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c120cae6"}
2026-07-13T03:31:21Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c120cae6"}, "namespace": "ai-tenant-e2e-mt-c120cae6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "12a53067-4561-4323-bdcc-1383efd567a1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c120cae6"}
2026-07-13T03:31:53Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c120cae6"}, "namespace": "ai-tenant-e2e-mt-c120cae6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a59d3629-6a58-4d52-97b0-2f9d700a1b2b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c120cae6"}
2026-07-13T03:31:53Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c120cae6"}, "namespace": "ai-tenant-e2e-mt-c120cae6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a59d3629-6a58-4d52-97b0-2f9d700a1b2b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c120cae6", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:31:53Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c120cae6"}, "namespace": "ai-tenant-e2e-mt-c120cae6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a59d3629-6a58-4d52-97b0-2f9d700a1b2b", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-c120cae6 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-13T03:31:53Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c120cae6"}, "namespace": "ai-tenant-e2e-mt-c120cae6", "name": "odh-trusted-ca-bundle", "reconcileID": "00c00276-581c-4b3c-bcda-25af56bfbbed", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c120cae6"}
2026-07-13T03:31:53Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c120cae6"}, "namespace": "ai-tenant-e2e-mt-c120cae6", "name": "odh-trusted-ca-bundle", "reconcileID": "00c00276-581c-4b3c-bcda-25af56bfbbed", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c120cae6"}
2026-07-13T03:31:53Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c120cae6"}, "namespace": "ai-tenant-e2e-mt-c120cae6", "name": "openshift-service-ca.crt", "reconcileID": "de668ee3-75ad-41dd-b495-5cad7a179283", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c120cae6"}
2026-07-13T03:31:53Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c120cae6"}, "namespace": "ai-tenant-e2e-mt-c120cae6", "name": "openshift-service-ca.crt", "reconcileID": "de668ee3-75ad-41dd-b495-5cad7a179283", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c120cae6"}
2026-07-13T03:31:53Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c120cae6"}, "namespace": "ai-tenant-e2e-mt-c120cae6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e8aa0a51-923b-4b24-9907-01d13b321403", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c120cae6"}
2026-07-13T03:31:53Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c120cae6"}, "namespace": "ai-tenant-e2e-mt-c120cae6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e8aa0a51-923b-4b24-9907-01d13b321403", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c120cae6"}
2026-07-13T03:31:54Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-6bb55d11"}, "namespace": "e2e-unlabeled-6bb55d11", "name": "openshift-service-ca.crt", "reconcileID": "f3feacef-503b-40a5-bb40-a57a945d3a39", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-6bb55d11"}
2026-07-13T03:31:54Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-6bb55d11"}, "namespace": "e2e-unlabeled-6bb55d11", "name": "odh-trusted-ca-bundle", "reconcileID": "a41f356b-ebb0-489e-9c57-71b7a5305474", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-6bb55d11"}
2026-07-13T03:31:54Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-6bb55d11"}, "namespace": "e2e-unlabeled-6bb55d11", "name": "openshift-service-ca.crt", "reconcileID": "de734427-6ef5-4fa1-87dc-7676f2db9395", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-6bb55d11", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:31:54Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-6bb55d11"}, "namespace": "e2e-unlabeled-6bb55d11", "name": "openshift-service-ca.crt", "reconcileID": "de734427-6ef5-4fa1-87dc-7676f2db9395", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-6bb55d11"}
2026-07-13T03:31:54Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-6bb55d11"}, "namespace": "e2e-unlabeled-6bb55d11", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0b5b08fa-4c98-4e87-b1b7-37cedaaf76a2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-6bb55d11"}
2026-07-13T03:31:54Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-6bb55d11"}, "namespace": "e2e-unlabeled-6bb55d11", "name": "openshift-service-ca.crt", "reconcileID": "c252f722-b0dc-49b3-a944-2c8369842a09", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-6bb55d11"}
2026-07-13T03:31:54Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-6bb55d11"}, "namespace": "e2e-unlabeled-6bb55d11", "name": "odh-trusted-ca-bundle", "reconcileID": "e5526e4a-f33c-4028-a8e5-bb2edb744245", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-6bb55d11", "update": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:31:54Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-6bb55d11"}, "namespace": "e2e-unlabeled-6bb55d11", "name": "odh-trusted-ca-bundle", "reconcileID": "e5526e4a-f33c-4028-a8e5-bb2edb744245", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-6bb55d11"}
2026-07-13T03:31:54Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-6bb55d11"}, "namespace": "e2e-unlabeled-6bb55d11", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "21ad6639-61b2-4002-a5a3-5f4b90fc89bf", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-6bb55d11"}
2026-07-13T03:32:15Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-6bb55d11"}, "namespace": "e2e-unlabeled-6bb55d11", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bd24933d-855e-400b-9370-b520305b1395", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-6bb55d11"}
2026-07-13T03:32:15Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-6bb55d11"}, "namespace": "e2e-unlabeled-6bb55d11", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bd24933d-855e-400b-9370-b520305b1395", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-6bb55d11", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:32:15Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-6bb55d11"}, "namespace": "e2e-unlabeled-6bb55d11", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bd24933d-855e-400b-9370-b520305b1395", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-unlabeled-6bb55d11 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-13T03:32:15Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-6bb55d11"}, "namespace": "e2e-unlabeled-6bb55d11", "name": "odh-trusted-ca-bundle", "reconcileID": "bc8a61db-d11d-4f16-ae6c-247f8802e344", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-6bb55d11"}
2026-07-13T03:32:15Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-6bb55d11"}, "namespace": "e2e-unlabeled-6bb55d11", "name": "odh-trusted-ca-bundle", "reconcileID": "bc8a61db-d11d-4f16-ae6c-247f8802e344", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-6bb55d11"}
2026-07-13T03:32:15Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-6bb55d11"}, "namespace": "e2e-unlabeled-6bb55d11", "name": "openshift-service-ca.crt", "reconcileID": "ae732254-1910-4a56-bbd2-356d24c261a9", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-6bb55d11"}
2026-07-13T03:32:15Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-6bb55d11"}, "namespace": "e2e-unlabeled-6bb55d11", "name": "openshift-service-ca.crt", "reconcileID": "ae732254-1910-4a56-bbd2-356d24c261a9", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-6bb55d11"}
2026-07-13T03:32:15Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-6bb55d11"}, "namespace": "e2e-unlabeled-6bb55d11", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d2dcc9f4-c712-4589-8759-749800a634bf", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-6bb55d11"}
2026-07-13T03:32:15Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-6bb55d11"}, "namespace": "e2e-unlabeled-6bb55d11", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d2dcc9f4-c712-4589-8759-749800a634bf", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-6bb55d11"}
2026-07-13T03:32:15Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-5b92e04a"}, "namespace": "ai-tenant-e2e-mt-5b92e04a", "name": "openshift-service-ca.crt", "reconcileID": "3dad6f56-c512-4d6b-98fd-62d1eb1c7e9a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-5b92e04a"}
2026-07-13T03:32:15Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-5b92e04a"}, "namespace": "ai-tenant-e2e-mt-5b92e04a", "name": "odh-trusted-ca-bundle", "reconcileID": "64c6c70a-19bb-4e58-b6b1-096144a8bd83", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-5b92e04a"}
2026-07-13T03:32:15Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-5b92e04a"}, "namespace": "ai-tenant-e2e-mt-5b92e04a", "name": "openshift-service-ca.crt", "reconcileID": "4d7e06fc-04f4-4b31-bc00-2b361f86a94c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-5b92e04a", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:32:15Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-5b92e04a"}, "namespace": "ai-tenant-e2e-mt-5b92e04a", "name": "openshift-service-ca.crt", "reconcileID": "4d7e06fc-04f4-4b31-bc00-2b361f86a94c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-5b92e04a"}
2026-07-13T03:32:15Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-5b92e04a"}, "namespace": "ai-tenant-e2e-mt-5b92e04a", "name": "openshift-service-ca.crt", "reconcileID": "bff772c3-6d2f-4438-aa43-7a06c7d7203a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-5b92e04a"}
2026-07-13T03:32:15Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-5b92e04a"}, "namespace": "ai-tenant-e2e-mt-5b92e04a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bc7d6325-cc23-4540-a75c-bb797a4d124e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-5b92e04a"}
2026-07-13T03:32:15Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-5b92e04a"}, "namespace": "ai-tenant-e2e-mt-5b92e04a", "name": "odh-trusted-ca-bundle", "reconcileID": "a84c057d-5689-42ab-a051-67fbd25ceef5", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-5b92e04a", "update": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:32:15Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-5b92e04a"}, "namespace": "ai-tenant-e2e-mt-5b92e04a", "name": "odh-trusted-ca-bundle", "reconcileID": "a84c057d-5689-42ab-a051-67fbd25ceef5", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-5b92e04a"}
2026-07-13T03:32:15Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-5b92e04a"}, "namespace": "ai-tenant-e2e-mt-5b92e04a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "82413e5e-4c54-40e9-ba92-5ef30e086730", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-5b92e04a"}
2026-07-13T03:32:32Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-5b92e04a"}, "namespace": "ai-tenant-e2e-mt-5b92e04a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "27d55de5-8741-4044-a1bd-040650ee4431", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-5b92e04a"}
2026-07-13T03:32:32Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-5b92e04a"}, "namespace": "ai-tenant-e2e-mt-5b92e04a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "27d55de5-8741-4044-a1bd-040650ee4431", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-5b92e04a", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:32:32Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-5b92e04a"}, "namespace": "ai-tenant-e2e-mt-5b92e04a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "27d55de5-8741-4044-a1bd-040650ee4431", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-5b92e04a because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-13T03:32:32Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-5b92e04a"}, "namespace": "ai-tenant-e2e-mt-5b92e04a", "name": "odh-trusted-ca-bundle", "reconcileID": "23c606db-2d65-4080-810f-74168fc3626a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-5b92e04a"}
2026-07-13T03:32:32Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-5b92e04a"}, "namespace": "ai-tenant-e2e-mt-5b92e04a", "name": "odh-trusted-ca-bundle", "reconcileID": "23c606db-2d65-4080-810f-74168fc3626a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-5b92e04a"}
2026-07-13T03:32:32Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-5b92e04a"}, "namespace": "ai-tenant-e2e-mt-5b92e04a", "name": "openshift-service-ca.crt", "reconcileID": "90bc7cce-4131-465e-be5c-b325d5057af7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-5b92e04a"}
2026-07-13T03:32:32Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-5b92e04a"}, "namespace": "ai-tenant-e2e-mt-5b92e04a", "name": "openshift-service-ca.crt", "reconcileID": "90bc7cce-4131-465e-be5c-b325d5057af7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-5b92e04a"}
2026-07-13T03:32:32Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-5b92e04a"}, "namespace": "ai-tenant-e2e-mt-5b92e04a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "09fe4c22-1781-4b20-bcce-6c43f0823199", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-5b92e04a"}
2026-07-13T03:32:32Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-5b92e04a"}, "namespace": "ai-tenant-e2e-mt-5b92e04a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "09fe4c22-1781-4b20-bcce-6c43f0823199", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-5b92e04a"}
2026-07-13T03:32:32Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-376fe730"}, "namespace": "ai-tenant-e2e-mt-376fe730", "name": "openshift-service-ca.crt", "reconcileID": "58260d2b-1eec-48b7-8530-c50b8a8157ea", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-376fe730"}
2026-07-13T03:32:32Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-376fe730"}, "namespace": "ai-tenant-e2e-mt-376fe730", "name": "odh-trusted-ca-bundle", "reconcileID": "22c3fe7d-d2fa-47ed-a2f5-dbc31221cd47", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-376fe730"}
2026-07-13T03:32:32Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-376fe730"}, "namespace": "ai-tenant-e2e-mt-376fe730", "name": "openshift-service-ca.crt", "reconcileID": "dabaaefc-a201-4868-8bd0-ce5b5d92492e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-376fe730", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:32:33Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-376fe730"}, "namespace": "ai-tenant-e2e-mt-376fe730", "name": "openshift-service-ca.crt", "reconcileID": "dabaaefc-a201-4868-8bd0-ce5b5d92492e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-376fe730"}
2026-07-13T03:32:33Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-376fe730"}, "namespace": "ai-tenant-e2e-mt-376fe730", "name": "openshift-service-ca.crt", "reconcileID": "28defda2-7844-44ce-92e0-7afd5a5ac83f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-376fe730"}
2026-07-13T03:32:33Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-376fe730"}, "namespace": "ai-tenant-e2e-mt-376fe730", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e5abaebc-091b-49fa-8cbb-f091d1f0b933", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-376fe730"}
2026-07-13T03:32:33Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-376fe730"}, "namespace": "ai-tenant-e2e-mt-376fe730", "name": "odh-trusted-ca-bundle", "reconcileID": "78f92dc9-d838-46c6-9da9-3d5a8d8c713e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-376fe730", "update": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:32:33Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-376fe730"}, "namespace": "ai-tenant-e2e-mt-376fe730", "name": "odh-trusted-ca-bundle", "reconcileID": "78f92dc9-d838-46c6-9da9-3d5a8d8c713e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-376fe730"}
2026-07-13T03:32:33Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-376fe730"}, "namespace": "ai-tenant-e2e-mt-376fe730", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "585050cd-e10e-49eb-ad72-3621215cb2f1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-376fe730"}
2026-07-13T03:32:33Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-60a9a173"}, "namespace": "ai-tenant-e2e-mt-60a9a173", "name": "openshift-service-ca.crt", "reconcileID": "2454c1e6-aa06-4128-9548-ef672ee97a76", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-60a9a173"}
2026-07-13T03:32:33Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-60a9a173"}, "namespace": "ai-tenant-e2e-mt-60a9a173", "name": "odh-trusted-ca-bundle", "reconcileID": "997a5e1a-dcd5-4934-b498-9f04b5b54d24", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-60a9a173"}
2026-07-13T03:32:33Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-60a9a173"}, "namespace": "ai-tenant-e2e-mt-60a9a173", "name": "openshift-service-ca.crt", "reconcileID": "db5ef02b-f3ea-45df-b6c7-7fe5ea703950", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-60a9a173", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:32:33Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-60a9a173"}, "namespace": "ai-tenant-e2e-mt-60a9a173", "name": "openshift-service-ca.crt", "reconcileID": "db5ef02b-f3ea-45df-b6c7-7fe5ea703950", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-60a9a173"}
2026-07-13T03:32:33Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-60a9a173"}, "namespace": "ai-tenant-e2e-mt-60a9a173", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "56fb76ae-4a55-4f4f-8c1f-15429e9eaae9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-60a9a173"}
2026-07-13T03:32:33Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-60a9a173"}, "namespace": "ai-tenant-e2e-mt-60a9a173", "name": "openshift-service-ca.crt", "reconcileID": "6e6555ab-944a-473c-91d7-b08c4b390377", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-60a9a173"}
2026-07-13T03:32:33Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-60a9a173"}, "namespace": "ai-tenant-e2e-mt-60a9a173", "name": "odh-trusted-ca-bundle", "reconcileID": "c5f90580-3e2b-4831-9bb1-f9efd31f204a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-60a9a173", "update": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:32:33Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-60a9a173"}, "namespace": "ai-tenant-e2e-mt-60a9a173", "name": "odh-trusted-ca-bundle", "reconcileID": "c5f90580-3e2b-4831-9bb1-f9efd31f204a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-60a9a173"}
2026-07-13T03:32:33Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-60a9a173"}, "namespace": "ai-tenant-e2e-mt-60a9a173", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "35e91a78-0c3e-4400-9814-ff8de0e52d2d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-60a9a173"}
2026-07-13T03:32:40Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-376fe730"}, "namespace": "ai-tenant-e2e-mt-376fe730", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2ef8bb40-debf-4627-961a-30c8aaaaf21a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-376fe730"}
2026-07-13T03:32:40Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-376fe730"}, "namespace": "ai-tenant-e2e-mt-376fe730", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2ef8bb40-debf-4627-961a-30c8aaaaf21a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-376fe730", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:32:40Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-376fe730"}, "namespace": "ai-tenant-e2e-mt-376fe730", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2ef8bb40-debf-4627-961a-30c8aaaaf21a", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-376fe730 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-13T03:32:40Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-376fe730"}, "namespace": "ai-tenant-e2e-mt-376fe730", "name": "odh-trusted-ca-bundle", "reconcileID": "52703267-ae91-47ca-9957-5608e47a785c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-376fe730"}
2026-07-13T03:32:40Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-376fe730"}, "namespace": "ai-tenant-e2e-mt-376fe730", "name": "odh-trusted-ca-bundle", "reconcileID": "52703267-ae91-47ca-9957-5608e47a785c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-376fe730"}
2026-07-13T03:32:40Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-376fe730"}, "namespace": "ai-tenant-e2e-mt-376fe730", "name": "openshift-service-ca.crt", "reconcileID": "fed95a68-d864-445f-bccb-b47e6f988112", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-376fe730"}
2026-07-13T03:32:40Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-376fe730"}, "namespace": "ai-tenant-e2e-mt-376fe730", "name": "openshift-service-ca.crt", "reconcileID": "fed95a68-d864-445f-bccb-b47e6f988112", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-376fe730"}
2026-07-13T03:32:40Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-376fe730"}, "namespace": "ai-tenant-e2e-mt-376fe730", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d6a07ab3-159e-4a15-823c-1b0cb5e2a0e2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-376fe730"}
2026-07-13T03:32:40Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-376fe730"}, "namespace": "ai-tenant-e2e-mt-376fe730", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d6a07ab3-159e-4a15-823c-1b0cb5e2a0e2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-376fe730"}
2026-07-13T03:32:47Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-60a9a173"}, "namespace": "ai-tenant-e2e-mt-60a9a173", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5ac35085-f073-48b7-bc38-bc6de70539db", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-60a9a173"}
2026-07-13T03:32:47Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-60a9a173"}, "namespace": "ai-tenant-e2e-mt-60a9a173", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5ac35085-f073-48b7-bc38-bc6de70539db", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-60a9a173", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:32:47Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-60a9a173"}, "namespace": "ai-tenant-e2e-mt-60a9a173", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5ac35085-f073-48b7-bc38-bc6de70539db", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-60a9a173 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-13T03:32:47Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-60a9a173"}, "namespace": "ai-tenant-e2e-mt-60a9a173", "name": "odh-trusted-ca-bundle", "reconcileID": "bee7f6b6-0085-409c-919e-1f588d67329e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-60a9a173"}
2026-07-13T03:32:47Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-60a9a173"}, "namespace": "ai-tenant-e2e-mt-60a9a173", "name": "odh-trusted-ca-bundle", "reconcileID": "bee7f6b6-0085-409c-919e-1f588d67329e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-60a9a173"}
2026-07-13T03:32:47Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-60a9a173"}, "namespace": "ai-tenant-e2e-mt-60a9a173", "name": "openshift-service-ca.crt", "reconcileID": "0a711ef9-d777-49eb-9b89-6ceb986a25fb", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-60a9a173"}
2026-07-13T03:32:47Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-60a9a173"}, "namespace": "ai-tenant-e2e-mt-60a9a173", "name": "openshift-service-ca.crt", "reconcileID": "0a711ef9-d777-49eb-9b89-6ceb986a25fb", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-60a9a173"}
2026-07-13T03:32:47Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-60a9a173"}, "namespace": "ai-tenant-e2e-mt-60a9a173", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d27d2bcc-7108-4c47-9f84-0639a197efab", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-60a9a173"}
2026-07-13T03:32:47Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-60a9a173"}, "namespace": "ai-tenant-e2e-mt-60a9a173", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d27d2bcc-7108-4c47-9f84-0639a197efab", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-60a9a173"}
2026-07-13T03:32:48Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a8f491a4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4", "reconcileID": "30f0fa03-83e2-4ad6-a37f-bfe82f1e4bde", "Gateway": "e2e-mt-a8f491a4", "namespace": "openshift-ingress"}
2026-07-13T03:32:48Z	INFO	Creating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a8f491a4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4", "reconcileID": "30f0fa03-83e2-4ad6-a37f-bfe82f1e4bde", "Gateway": "e2e-mt-a8f491a4", "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4-authn-ssl"}
2026-07-13T03:32:48Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a8f491a4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4", "reconcileID": "30f0fa03-83e2-4ad6-a37f-bfe82f1e4bde", "Gateway": "e2e-mt-a8f491a4", "namespace": "openshift-ingress"}
2026-07-13T03:32:48Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a8f491a4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4", "reconcileID": "4dd9e2bf-ddfa-4702-a0f8-28cb61e8d852", "Gateway": "e2e-mt-a8f491a4", "namespace": "openshift-ingress"}
2026-07-13T03:32:48Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a8f491a4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4", "reconcileID": "4dd9e2bf-ddfa-4702-a0f8-28cb61e8d852", "Gateway": "e2e-mt-a8f491a4", "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4-authn-ssl"}
2026-07-13T03:32:48Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a8f491a4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4", "reconcileID": "4dd9e2bf-ddfa-4702-a0f8-28cb61e8d852", "Gateway": "e2e-mt-a8f491a4", "namespace": "openshift-ingress"}
2026-07-13T03:32:48Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a8f491a4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4", "reconcileID": "adaf108c-fbf8-4eb9-a8b0-ae14c29a079e", "Gateway": "e2e-mt-a8f491a4", "namespace": "openshift-ingress"}
2026-07-13T03:32:48Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a8f491a4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4", "reconcileID": "adaf108c-fbf8-4eb9-a8b0-ae14c29a079e", "Gateway": "e2e-mt-a8f491a4", "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4-authn-ssl"}
2026-07-13T03:32:48Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a8f491a4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4", "reconcileID": "adaf108c-fbf8-4eb9-a8b0-ae14c29a079e", "Gateway": "e2e-mt-a8f491a4", "namespace": "openshift-ingress"}
2026-07-13T03:32:48Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a8f491a4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4", "reconcileID": "63c1a92f-209e-4b11-8476-29444d984bfb", "Gateway": "e2e-mt-a8f491a4", "namespace": "openshift-ingress"}
2026-07-13T03:32:48Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a8f491a4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4", "reconcileID": "63c1a92f-209e-4b11-8476-29444d984bfb", "Gateway": "e2e-mt-a8f491a4", "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4-authn-ssl"}
2026-07-13T03:32:48Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a8f491a4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4", "reconcileID": "63c1a92f-209e-4b11-8476-29444d984bfb", "Gateway": "e2e-mt-a8f491a4", "namespace": "openshift-ingress"}
2026-07-13T03:32:48Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a8f491a4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4", "reconcileID": "f0a20ea2-4f3c-47e8-8b7d-11433869f7c5", "Gateway": "e2e-mt-a8f491a4", "namespace": "openshift-ingress"}
2026-07-13T03:32:48Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a8f491a4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4", "reconcileID": "f0a20ea2-4f3c-47e8-8b7d-11433869f7c5", "Gateway": "e2e-mt-a8f491a4", "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4-authn-ssl"}
2026-07-13T03:32:48Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a8f491a4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4", "reconcileID": "f0a20ea2-4f3c-47e8-8b7d-11433869f7c5", "Gateway": "e2e-mt-a8f491a4", "namespace": "openshift-ingress"}
2026-07-13T03:32:48Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a8f491a4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4", "reconcileID": "683bf73f-e2d5-461f-9c2f-5467921113b9", "Gateway": "e2e-mt-a8f491a4", "namespace": "openshift-ingress"}
2026-07-13T03:32:48Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a8f491a4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4", "reconcileID": "683bf73f-e2d5-461f-9c2f-5467921113b9", "Gateway": "e2e-mt-a8f491a4", "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4-authn-ssl"}
2026-07-13T03:32:48Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a8f491a4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4", "reconcileID": "683bf73f-e2d5-461f-9c2f-5467921113b9", "Gateway": "e2e-mt-a8f491a4", "namespace": "openshift-ingress"}
2026-07-13T03:32:50Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a8f491a4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4", "reconcileID": "27bbc353-cef7-442c-ba87-be273edbe9bb", "Gateway": "e2e-mt-a8f491a4", "namespace": "openshift-ingress"}
2026-07-13T03:32:50Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a8f491a4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4", "reconcileID": "27bbc353-cef7-442c-ba87-be273edbe9bb", "Gateway": "e2e-mt-a8f491a4", "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4-authn-ssl"}
2026-07-13T03:32:50Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-a8f491a4","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-a8f491a4", "reconcileID": "27bbc353-cef7-442c-ba87-be273edbe9bb", "Gateway": "e2e-mt-a8f491a4", "namespace": "openshift-ingress"}
2026-07-13T03:32:51Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a8f491a4"}, "namespace": "ai-tenant-e2e-mt-a8f491a4", "name": "openshift-service-ca.crt", "reconcileID": "cc2d76e0-aa95-44a1-b176-fa6609fddac0", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a8f491a4"}
2026-07-13T03:32:51Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a8f491a4"}, "namespace": "ai-tenant-e2e-mt-a8f491a4", "name": "odh-trusted-ca-bundle", "reconcileID": "b703e86e-48c7-47f3-a6bb-20b5c3a12096", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a8f491a4"}
2026-07-13T03:32:51Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a8f491a4"}, "namespace": "ai-tenant-e2e-mt-a8f491a4", "name": "openshift-service-ca.crt", "reconcileID": "5f3fd6d8-5c03-4c6c-8a70-40e71a6b3eba", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a8f491a4", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:32:51Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a8f491a4"}, "namespace": "ai-tenant-e2e-mt-a8f491a4", "name": "openshift-service-ca.crt", "reconcileID": "5f3fd6d8-5c03-4c6c-8a70-40e71a6b3eba", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a8f491a4"}
2026-07-13T03:32:51Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a8f491a4"}, "namespace": "ai-tenant-e2e-mt-a8f491a4", "name": "openshift-service-ca.crt", "reconcileID": "6d1d984b-5dab-4e2f-a8ce-0d64fc6e3186", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a8f491a4"}
2026-07-13T03:32:51Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a8f491a4"}, "namespace": "ai-tenant-e2e-mt-a8f491a4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2076d527-eaa7-4408-9aea-a16fb269d3f4", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a8f491a4"}
2026-07-13T03:32:51Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a8f491a4"}, "namespace": "ai-tenant-e2e-mt-a8f491a4", "name": "odh-trusted-ca-bundle", "reconcileID": "a82d4204-a1f6-4858-9d06-136d384a0a7f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a8f491a4", "update": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:32:51Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a8f491a4"}, "namespace": "ai-tenant-e2e-mt-a8f491a4", "name": "odh-trusted-ca-bundle", "reconcileID": "a82d4204-a1f6-4858-9d06-136d384a0a7f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a8f491a4"}
2026-07-13T03:32:51Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a8f491a4"}, "namespace": "ai-tenant-e2e-mt-a8f491a4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b0a8df3b-43ae-4714-a1ee-b6436d8f0381", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a8f491a4"}
2026-07-13T03:36:06Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a8f491a4"}, "namespace": "ai-tenant-e2e-mt-a8f491a4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6226aa3b-4ac2-48e7-97d4-55ad32255226", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a8f491a4"}
2026-07-13T03:36:06Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a8f491a4"}, "namespace": "ai-tenant-e2e-mt-a8f491a4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6226aa3b-4ac2-48e7-97d4-55ad32255226", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a8f491a4", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:36:06Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a8f491a4"}, "namespace": "ai-tenant-e2e-mt-a8f491a4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6226aa3b-4ac2-48e7-97d4-55ad32255226", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-a8f491a4 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-13T03:36:06Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a8f491a4"}, "namespace": "ai-tenant-e2e-mt-a8f491a4", "name": "odh-trusted-ca-bundle", "reconcileID": "304976d4-d89e-4f9e-9bb4-fe6eea3ab426", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a8f491a4"}
2026-07-13T03:36:06Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a8f491a4"}, "namespace": "ai-tenant-e2e-mt-a8f491a4", "name": "odh-trusted-ca-bundle", "reconcileID": "304976d4-d89e-4f9e-9bb4-fe6eea3ab426", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a8f491a4"}
2026-07-13T03:36:06Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a8f491a4"}, "namespace": "ai-tenant-e2e-mt-a8f491a4", "name": "openshift-service-ca.crt", "reconcileID": "7731155c-4d77-4695-85e7-473970ee253c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a8f491a4"}
2026-07-13T03:36:06Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a8f491a4"}, "namespace": "ai-tenant-e2e-mt-a8f491a4", "name": "openshift-service-ca.crt", "reconcileID": "7731155c-4d77-4695-85e7-473970ee253c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a8f491a4"}
2026-07-13T03:36:06Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a8f491a4"}, "namespace": "ai-tenant-e2e-mt-a8f491a4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0313cad7-6979-4165-b24e-06fa5ee24829", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a8f491a4"}
2026-07-13T03:36:06Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a8f491a4"}, "namespace": "ai-tenant-e2e-mt-a8f491a4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0313cad7-6979-4165-b24e-06fa5ee24829", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a8f491a4"}
2026-07-13T03:36:07Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-b5c4b62a"}, "namespace": "e2e-webhook-sub-b5c4b62a", "name": "openshift-service-ca.crt", "reconcileID": "5826fcff-1436-4c2f-9a6d-bb9ecba78c24", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-b5c4b62a"}
2026-07-13T03:36:07Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-b5c4b62a"}, "namespace": "e2e-webhook-sub-b5c4b62a", "name": "odh-trusted-ca-bundle", "reconcileID": "b6074938-2594-4ec4-bf2c-3bbc04511b83", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-b5c4b62a"}
2026-07-13T03:36:07Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-b5c4b62a"}, "namespace": "e2e-webhook-sub-b5c4b62a", "name": "openshift-service-ca.crt", "reconcileID": "537e2d1f-a3b0-4284-9abd-d498b9a7cf0e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-b5c4b62a", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:36:07Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-b5c4b62a"}, "namespace": "e2e-webhook-sub-b5c4b62a", "name": "openshift-service-ca.crt", "reconcileID": "537e2d1f-a3b0-4284-9abd-d498b9a7cf0e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-b5c4b62a"}
2026-07-13T03:36:07Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-b5c4b62a"}, "namespace": "e2e-webhook-sub-b5c4b62a", "name": "openshift-service-ca.crt", "reconcileID": "3dc9cf87-3c2d-4395-ba0d-f66e5781f789", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-b5c4b62a"}
2026-07-13T03:36:07Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-b5c4b62a"}, "namespace": "e2e-webhook-sub-b5c4b62a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3fe86ad6-1b24-4dbd-b988-8b0f35c75624", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-b5c4b62a"}
2026-07-13T03:36:07Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-b5c4b62a"}, "namespace": "e2e-webhook-sub-b5c4b62a", "name": "odh-trusted-ca-bundle", "reconcileID": "9ff4b83d-8484-4391-b950-3a4d52e6c89d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-b5c4b62a", "update": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:36:08Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-b5c4b62a"}, "namespace": "e2e-webhook-sub-b5c4b62a", "name": "odh-trusted-ca-bundle", "reconcileID": "9ff4b83d-8484-4391-b950-3a4d52e6c89d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-b5c4b62a"}
2026-07-13T03:36:08Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-b5c4b62a"}, "namespace": "e2e-webhook-sub-b5c4b62a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e7b2b05a-38c8-47d0-8d0c-d3357ca7a405", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-b5c4b62a"}
2026-07-13T03:36:13Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-b5c4b62a"}, "namespace": "e2e-webhook-sub-b5c4b62a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9129191e-e792-4531-9d62-fee8203f5541", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-b5c4b62a"}
2026-07-13T03:36:13Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-b5c4b62a"}, "namespace": "e2e-webhook-sub-b5c4b62a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9129191e-e792-4531-9d62-fee8203f5541", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-b5c4b62a", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:36:13Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-b5c4b62a"}, "namespace": "e2e-webhook-sub-b5c4b62a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9129191e-e792-4531-9d62-fee8203f5541", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-sub-b5c4b62a because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-13T03:36:13Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-b5c4b62a"}, "namespace": "e2e-webhook-sub-b5c4b62a", "name": "odh-trusted-ca-bundle", "reconcileID": "6494d780-76d2-412a-a5ba-2bad5e069518", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-b5c4b62a"}
2026-07-13T03:36:13Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-b5c4b62a"}, "namespace": "e2e-webhook-sub-b5c4b62a", "name": "odh-trusted-ca-bundle", "reconcileID": "6494d780-76d2-412a-a5ba-2bad5e069518", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-b5c4b62a"}
2026-07-13T03:36:13Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-b5c4b62a"}, "namespace": "e2e-webhook-sub-b5c4b62a", "name": "openshift-service-ca.crt", "reconcileID": "6262fcd4-3544-49ba-abf5-5bff8e052272", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-b5c4b62a"}
2026-07-13T03:36:13Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-b5c4b62a"}, "namespace": "e2e-webhook-sub-b5c4b62a", "name": "openshift-service-ca.crt", "reconcileID": "6262fcd4-3544-49ba-abf5-5bff8e052272", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-b5c4b62a"}
2026-07-13T03:36:13Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-b5c4b62a"}, "namespace": "e2e-webhook-sub-b5c4b62a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "244577a9-5829-4965-a100-78df610b44b6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-b5c4b62a"}
2026-07-13T03:36:13Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-b5c4b62a"}, "namespace": "e2e-webhook-sub-b5c4b62a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "244577a9-5829-4965-a100-78df610b44b6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-b5c4b62a"}
2026-07-13T03:36:14Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-694d09d0"}, "namespace": "e2e-webhook-auth-694d09d0", "name": "openshift-service-ca.crt", "reconcileID": "e2175779-d5d7-4969-bb04-47cd2bf61edd", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-694d09d0"}
2026-07-13T03:36:14Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-694d09d0"}, "namespace": "e2e-webhook-auth-694d09d0", "name": "odh-trusted-ca-bundle", "reconcileID": "215eb39c-9e21-4ea1-9288-a2f4d429e365", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-694d09d0"}
2026-07-13T03:36:14Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-694d09d0"}, "namespace": "e2e-webhook-auth-694d09d0", "name": "openshift-service-ca.crt", "reconcileID": "80ee0edf-f2bc-4cf5-8c7a-5efc61823b1d", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-694d09d0", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:36:14Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-694d09d0"}, "namespace": "e2e-webhook-auth-694d09d0", "name": "openshift-service-ca.crt", "reconcileID": "80ee0edf-f2bc-4cf5-8c7a-5efc61823b1d", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-694d09d0"}
2026-07-13T03:36:14Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-694d09d0"}, "namespace": "e2e-webhook-auth-694d09d0", "name": "openshift-service-ca.crt", "reconcileID": "489ff9fe-0911-4ca7-a76b-96d666cfd9ba", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-694d09d0"}
2026-07-13T03:36:14Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-694d09d0"}, "namespace": "e2e-webhook-auth-694d09d0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "755a611c-f533-4fd5-b77e-8d1bd53ef1f8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-694d09d0"}
2026-07-13T03:36:14Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-694d09d0"}, "namespace": "e2e-webhook-auth-694d09d0", "name": "odh-trusted-ca-bundle", "reconcileID": "3a156381-e4f0-4a76-a00e-8f6f12dd7e14", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-694d09d0", "update": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:36:14Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-694d09d0"}, "namespace": "e2e-webhook-auth-694d09d0", "name": "odh-trusted-ca-bundle", "reconcileID": "3a156381-e4f0-4a76-a00e-8f6f12dd7e14", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-694d09d0"}
2026-07-13T03:36:14Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-694d09d0"}, "namespace": "e2e-webhook-auth-694d09d0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f10e8023-a8bf-4d2b-958c-3aaf948a9006", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-694d09d0"}
2026-07-13T03:36:19Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-694d09d0"}, "namespace": "e2e-webhook-auth-694d09d0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cc54008d-430b-4db4-95c1-85b921838c75", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-694d09d0"}
2026-07-13T03:36:19Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-694d09d0"}, "namespace": "e2e-webhook-auth-694d09d0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cc54008d-430b-4db4-95c1-85b921838c75", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-694d09d0", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:36:19Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-694d09d0"}, "namespace": "e2e-webhook-auth-694d09d0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cc54008d-430b-4db4-95c1-85b921838c75", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-auth-694d09d0 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-13T03:36:19Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-694d09d0"}, "namespace": "e2e-webhook-auth-694d09d0", "name": "odh-trusted-ca-bundle", "reconcileID": "00222976-c172-4ccd-9a68-a860ba959a7d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-694d09d0"}
2026-07-13T03:36:19Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-694d09d0"}, "namespace": "e2e-webhook-auth-694d09d0", "name": "odh-trusted-ca-bundle", "reconcileID": "00222976-c172-4ccd-9a68-a860ba959a7d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-694d09d0"}
2026-07-13T03:36:19Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-694d09d0"}, "namespace": "e2e-webhook-auth-694d09d0", "name": "openshift-service-ca.crt", "reconcileID": "67d15a66-1fd8-42fb-8727-b68805ae73c4", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-694d09d0"}
2026-07-13T03:36:19Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-694d09d0"}, "namespace": "e2e-webhook-auth-694d09d0", "name": "openshift-service-ca.crt", "reconcileID": "67d15a66-1fd8-42fb-8727-b68805ae73c4", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-694d09d0"}
2026-07-13T03:36:19Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-694d09d0"}, "namespace": "e2e-webhook-auth-694d09d0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2a864ff3-9ec0-4eac-a3f6-e82af97df975", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-694d09d0"}
2026-07-13T03:36:19Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-694d09d0"}, "namespace": "e2e-webhook-auth-694d09d0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2a864ff3-9ec0-4eac-a3f6-e82af97df975", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-694d09d0"}
2026-07-13T03:36:23Z	INFO	No monitoring namespace detected, skipping monitoring reconciliation.	{"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"}
2026-07-13T03:36:23Z	INFO	Multi Node reconciling.	{"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"}
2026-07-13T03:36:25Z	INFO	ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt	{"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"}
2026-07-13T03:36:25Z	INFO	Multi Node reconciled successfully.	{"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"}
2026-07-13T03:36:27Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "b5475188-99b1-4393-9eb0-530e2ec705c9", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:36:27Z	INFO	Creating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "b5475188-99b1-4393-9eb0-530e2ec705c9", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress", "name": "e2e-mt-766a483d-authn-ssl"}
2026-07-13T03:36:27Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "b5475188-99b1-4393-9eb0-530e2ec705c9", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:36:27Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "3c5dbf2d-d6ed-4cdf-ac3b-c1fc57f5bc6f", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:36:27Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "3c5dbf2d-d6ed-4cdf-ac3b-c1fc57f5bc6f", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress", "name": "e2e-mt-766a483d-authn-ssl"}
2026-07-13T03:36:27Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "3c5dbf2d-d6ed-4cdf-ac3b-c1fc57f5bc6f", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:36:27Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "b8003b78-d570-44cf-8bba-1844fb5a7a0c", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:36:27Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "b8003b78-d570-44cf-8bba-1844fb5a7a0c", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress", "name": "e2e-mt-766a483d-authn-ssl"}
2026-07-13T03:36:27Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "b8003b78-d570-44cf-8bba-1844fb5a7a0c", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:36:27Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "86bfff7c-a8ad-469b-a84b-bccc6dfd4833", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:36:27Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "86bfff7c-a8ad-469b-a84b-bccc6dfd4833", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress", "name": "e2e-mt-766a483d-authn-ssl"}
2026-07-13T03:36:27Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "86bfff7c-a8ad-469b-a84b-bccc6dfd4833", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:36:27Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "ba51f68a-8137-48e2-af0e-c992099dddd0", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:36:27Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "ba51f68a-8137-48e2-af0e-c992099dddd0", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress", "name": "e2e-mt-766a483d-authn-ssl"}
2026-07-13T03:36:27Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "ba51f68a-8137-48e2-af0e-c992099dddd0", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:36:27Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "f089b7aa-4556-401c-9007-23a7d792bb3e", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:36:27Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "f089b7aa-4556-401c-9007-23a7d792bb3e", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress", "name": "e2e-mt-766a483d-authn-ssl"}
2026-07-13T03:36:27Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "f089b7aa-4556-401c-9007-23a7d792bb3e", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:36:28Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "b011dca0-8024-4ffe-9f48-d35ad3e29ad6", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:36:28Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "b011dca0-8024-4ffe-9f48-d35ad3e29ad6", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress", "name": "e2e-mt-766a483d-authn-ssl"}
2026-07-13T03:36:28Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "b011dca0-8024-4ffe-9f48-d35ad3e29ad6", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:36:30Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "openshift-service-ca.crt", "reconcileID": "9136f0f4-f927-42e0-8ec7-f93b7ac2848b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:36:30Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "odh-trusted-ca-bundle", "reconcileID": "5fce65a0-e240-4d45-b3da-6b5ffbf2e161", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:36:30Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "openshift-service-ca.crt", "reconcileID": "fb70132f-83bf-498c-bea3-04cfc35ea9d6", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-766a483d", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:36:30Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "openshift-service-ca.crt", "reconcileID": "fb70132f-83bf-498c-bea3-04cfc35ea9d6", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:36:30Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a6d37b99-5fec-4605-a542-89a7327996ca", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:36:30Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "openshift-service-ca.crt", "reconcileID": "0c25486d-e7e2-46b9-9bb8-c2f59486c113", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:36:30Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "odh-trusted-ca-bundle", "reconcileID": "253c5807-9337-42b7-b430-beb9987d47c6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-766a483d", "update": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:36:30Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "odh-trusted-ca-bundle", "reconcileID": "253c5807-9337-42b7-b430-beb9987d47c6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:36:30Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b680f0f8-7fd7-418a-978d-4cb8a2bbe668", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:36:30Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "37bdcc7f-f2ef-491d-a18f-7dea88ce10d0", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:36:30Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "37bdcc7f-f2ef-491d-a18f-7dea88ce10d0", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress", "name": "e2e-mt-766a483d-authn-ssl"}
2026-07-13T03:36:30Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "37bdcc7f-f2ef-491d-a18f-7dea88ce10d0", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:36:30Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "f044a0ad-afa2-4626-86a0-d7d8f052581c", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"}
2026-07-13T03:36:35Z	DEBUG	Reconciling LLMInferenceService sub-resources	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "4c780c61-b5b5-4a34-a3d4-1365ba2e7ec8", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:36:35Z	DEBUG	Starting AuthPolicy reconciliation for LLMInferenceService	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "4c780c61-b5b5-4a34-a3d4-1365ba2e7ec8", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:36:35Z	DEBUG	Reconciling HTTPRoute AuthPolicy	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "4c780c61-b5b5-4a34-a3d4-1365ba2e7ec8", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:36:35Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "c85fe61d-3e37-4eeb-8a1e-95202b11172a", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:36:35Z	DEBUG	Processing HTTPRoute AuthPolicy delta	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "4c780c61-b5b5-4a34-a3d4-1365ba2e7ec8", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d-kserve-route-authn"}
2026-07-13T03:36:35Z	DEBUG	AuthType	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "4c780c61-b5b5-4a34-a3d4-1365ba2e7ec8", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "authType": "userdefined"}
2026-07-13T03:36:35Z	DEBUG	Successfully reconciled LLMInferenceService	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "4c780c61-b5b5-4a34-a3d4-1365ba2e7ec8", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:36:35Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "c85fe61d-3e37-4eeb-8a1e-95202b11172a", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress", "name": "e2e-mt-766a483d-authn-ssl"}
2026-07-13T03:36:35Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "c85fe61d-3e37-4eeb-8a1e-95202b11172a", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:36:35Z	DEBUG	Reconciling LLMInferenceService sub-resources	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "bb7b1712-e682-4bd8-87af-0695681e30b0", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:36:35Z	DEBUG	Starting AuthPolicy reconciliation for LLMInferenceService	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "bb7b1712-e682-4bd8-87af-0695681e30b0", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:36:35Z	DEBUG	Reconciling HTTPRoute AuthPolicy	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "bb7b1712-e682-4bd8-87af-0695681e30b0", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:36:35Z	DEBUG	Processing HTTPRoute AuthPolicy delta	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "bb7b1712-e682-4bd8-87af-0695681e30b0", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d-kserve-route-authn"}
2026-07-13T03:36:35Z	DEBUG	AuthType	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "bb7b1712-e682-4bd8-87af-0695681e30b0", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "authType": "userdefined"}
2026-07-13T03:36:35Z	DEBUG	Successfully reconciled LLMInferenceService	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "bb7b1712-e682-4bd8-87af-0695681e30b0", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:36:35Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "f8710fe3-b141-4a74-8c83-37164408d808", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:36:35Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "f8710fe3-b141-4a74-8c83-37164408d808", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress", "name": "e2e-mt-766a483d-authn-ssl"}
2026-07-13T03:36:35Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "f8710fe3-b141-4a74-8c83-37164408d808", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:36:52Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "02b2b3ca-f967-412d-8d2d-60453cabc89a", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:36:52Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "02b2b3ca-f967-412d-8d2d-60453cabc89a", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress", "name": "e2e-mt-766a483d-authn-ssl"}
2026-07-13T03:36:52Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "02b2b3ca-f967-412d-8d2d-60453cabc89a", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:36:53Z	DEBUG	Reconciling LLMInferenceService sub-resources	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "5d7656b6-3227-45d8-a6cf-0f34f8df8cd1", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:36:53Z	DEBUG	Starting AuthPolicy reconciliation for LLMInferenceService	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "5d7656b6-3227-45d8-a6cf-0f34f8df8cd1", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:36:53Z	DEBUG	Reconciling HTTPRoute AuthPolicy	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "5d7656b6-3227-45d8-a6cf-0f34f8df8cd1", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:36:53Z	DEBUG	Processing HTTPRoute AuthPolicy delta	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "5d7656b6-3227-45d8-a6cf-0f34f8df8cd1", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d-kserve-route-authn"}
2026-07-13T03:36:53Z	DEBUG	AuthType	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "5d7656b6-3227-45d8-a6cf-0f34f8df8cd1", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "authType": "userdefined"}
2026-07-13T03:36:53Z	DEBUG	Successfully reconciled LLMInferenceService	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "5d7656b6-3227-45d8-a6cf-0f34f8df8cd1", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:36:57Z	DEBUG	Reconciling LLMInferenceService sub-resources	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "93ff8a65-8718-458a-b9fe-d7dba18b1bbb", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:36:57Z	DEBUG	Starting AuthPolicy reconciliation for LLMInferenceService	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "93ff8a65-8718-458a-b9fe-d7dba18b1bbb", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:36:57Z	DEBUG	Reconciling HTTPRoute AuthPolicy	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "93ff8a65-8718-458a-b9fe-d7dba18b1bbb", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:36:57Z	DEBUG	Processing HTTPRoute AuthPolicy delta	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "93ff8a65-8718-458a-b9fe-d7dba18b1bbb", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d-kserve-route-authn"}
2026-07-13T03:36:57Z	DEBUG	AuthType	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "93ff8a65-8718-458a-b9fe-d7dba18b1bbb", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "authType": "userdefined"}
2026-07-13T03:36:57Z	DEBUG	Successfully reconciled LLMInferenceService	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "93ff8a65-8718-458a-b9fe-d7dba18b1bbb", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:03Z	DEBUG	Reconciling LLMInferenceService sub-resources	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "e0df6efb-73ce-4a0b-a3e1-6ece014dda79", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:03Z	DEBUG	Starting AuthPolicy reconciliation for LLMInferenceService	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "e0df6efb-73ce-4a0b-a3e1-6ece014dda79", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:03Z	DEBUG	Reconciling HTTPRoute AuthPolicy	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "e0df6efb-73ce-4a0b-a3e1-6ece014dda79", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:03Z	DEBUG	Processing HTTPRoute AuthPolicy delta	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "e0df6efb-73ce-4a0b-a3e1-6ece014dda79", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d-kserve-route-authn"}
2026-07-13T03:37:03Z	DEBUG	AuthType	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "e0df6efb-73ce-4a0b-a3e1-6ece014dda79", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "authType": "userdefined"}
2026-07-13T03:37:03Z	DEBUG	Successfully reconciled LLMInferenceService	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "e0df6efb-73ce-4a0b-a3e1-6ece014dda79", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:11Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "ff249a8c-a10c-4f84-85ed-e261bbaf8599", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:37:11Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "ff249a8c-a10c-4f84-85ed-e261bbaf8599", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress", "name": "e2e-mt-766a483d-authn-ssl"}
2026-07-13T03:37:11Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "ff249a8c-a10c-4f84-85ed-e261bbaf8599", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:37:11Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "a6b2c489-9ebf-4f81-b63d-9a356d90c397", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:37:11Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "a6b2c489-9ebf-4f81-b63d-9a356d90c397", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress", "name": "e2e-mt-766a483d-authn-ssl"}
2026-07-13T03:37:11Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "a6b2c489-9ebf-4f81-b63d-9a356d90c397", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:37:11Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "f3dedfd4-f907-41ae-a8ca-bfb8e6a26297", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:37:11Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "f3dedfd4-f907-41ae-a8ca-bfb8e6a26297", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress", "name": "e2e-mt-766a483d-authn-ssl"}
2026-07-13T03:37:12Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "f3dedfd4-f907-41ae-a8ca-bfb8e6a26297", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:37:18Z	DEBUG	Reconciling LLMInferenceService sub-resources	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "2a3ad0f4-2fc2-457a-8614-cf4a6535e2a3", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:18Z	DEBUG	Starting AuthPolicy reconciliation for LLMInferenceService	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "2a3ad0f4-2fc2-457a-8614-cf4a6535e2a3", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:18Z	DEBUG	Reconciling HTTPRoute AuthPolicy	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "2a3ad0f4-2fc2-457a-8614-cf4a6535e2a3", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:18Z	DEBUG	Processing HTTPRoute AuthPolicy delta	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "2a3ad0f4-2fc2-457a-8614-cf4a6535e2a3", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d-kserve-route-authn"}
2026-07-13T03:37:18Z	DEBUG	AuthType	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "2a3ad0f4-2fc2-457a-8614-cf4a6535e2a3", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "authType": "userdefined"}
2026-07-13T03:37:18Z	DEBUG	Successfully reconciled LLMInferenceService	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "2a3ad0f4-2fc2-457a-8614-cf4a6535e2a3", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:54Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d2871e86-b296-412f-9ccf-4cb3650bebbe", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:54Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d2871e86-b296-412f-9ccf-4cb3650bebbe", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-766a483d", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:37:54Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d2871e86-b296-412f-9ccf-4cb3650bebbe", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-766a483d because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-13T03:37:54Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "odh-trusted-ca-bundle", "reconcileID": "6e64a590-413c-4fcb-8bb1-b36e55e229fe", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:54Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "odh-trusted-ca-bundle", "reconcileID": "6e64a590-413c-4fcb-8bb1-b36e55e229fe", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:54Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "openshift-service-ca.crt", "reconcileID": "0aec3a58-6055-48f6-a638-f2e0146057d7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:54Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "openshift-service-ca.crt", "reconcileID": "0aec3a58-6055-48f6-a638-f2e0146057d7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:54Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a29d69f1-5e4b-4695-b704-d1c7319706fe", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:54Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a29d69f1-5e4b-4695-b704-d1c7319706fe", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:54Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "cbfecaf3-8afd-4a32-af57-07b5ad3c7c7c", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:37:54Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "cbfecaf3-8afd-4a32-af57-07b5ad3c7c7c", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress", "name": "e2e-mt-766a483d-authn-ssl"}
2026-07-13T03:37:54Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "cbfecaf3-8afd-4a32-af57-07b5ad3c7c7c", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:37:54Z	DEBUG	Reconciling LLMInferenceService sub-resources	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "1bf064f6-8d15-4689-bdc4-06f25d22e5c9", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:54Z	DEBUG	Starting AuthPolicy reconciliation for LLMInferenceService	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "1bf064f6-8d15-4689-bdc4-06f25d22e5c9", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:54Z	DEBUG	Reconciling HTTPRoute AuthPolicy	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "1bf064f6-8d15-4689-bdc4-06f25d22e5c9", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:54Z	DEBUG	Processing HTTPRoute AuthPolicy delta	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "1bf064f6-8d15-4689-bdc4-06f25d22e5c9", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d-kserve-route-authn"}
2026-07-13T03:37:54Z	DEBUG	AuthType	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "1bf064f6-8d15-4689-bdc4-06f25d22e5c9", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "authType": "userdefined"}
2026-07-13T03:37:54Z	DEBUG	Successfully reconciled LLMInferenceService	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "1bf064f6-8d15-4689-bdc4-06f25d22e5c9", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:54Z	INFO	LLMInferenceService being deleted, cleaning up sub-resources	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "21cedb18-981c-4a50-8076-75d800d00f75", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:54Z	DEBUG	Triggering Delete for LLMInferenceService	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "21cedb18-981c-4a50-8076-75d800d00f75", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d"}
2026-07-13T03:37:54Z	DEBUG	Deleting AuthPolicies for LLMInferenceService	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "21cedb18-981c-4a50-8076-75d800d00f75", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:54Z	DEBUG	Deleting HTTPRoute AuthPolicy for LLMInferenceService	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "21cedb18-981c-4a50-8076-75d800d00f75", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:54Z	DEBUG	Attempting to delete HTTPRoute AuthPolicy	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "21cedb18-981c-4a50-8076-75d800d00f75", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d-kserve-route-authn"}
2026-07-13T03:37:54Z	INFO	Successfully deleted HTTPRoute AuthPolicy	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "21cedb18-981c-4a50-8076-75d800d00f75", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d-kserve-route-authn"}
2026-07-13T03:37:54Z	DEBUG	Triggering LLMInferenceService Cleanup for Namespace	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "21cedb18-981c-4a50-8076-75d800d00f75", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:37:54Z	DEBUG	Cleaning up LLMInferenceService sub-resources	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "21cedb18-981c-4a50-8076-75d800d00f75", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:38:06Z	INFO	LLMInferenceService being deleted, cleaning up sub-resources	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "a2bddab3-ee8f-4e22-b407-beba508677b0", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:38:06Z	DEBUG	Triggering Delete for LLMInferenceService	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "a2bddab3-ee8f-4e22-b407-beba508677b0", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d"}
2026-07-13T03:38:06Z	DEBUG	Deleting AuthPolicies for LLMInferenceService	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "a2bddab3-ee8f-4e22-b407-beba508677b0", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:38:06Z	DEBUG	Deleting HTTPRoute AuthPolicy for LLMInferenceService	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "a2bddab3-ee8f-4e22-b407-beba508677b0", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:38:06Z	DEBUG	Attempting to delete HTTPRoute AuthPolicy	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "a2bddab3-ee8f-4e22-b407-beba508677b0", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d-kserve-route-authn"}
2026-07-13T03:38:06Z	INFO	Successfully deleted HTTPRoute AuthPolicy	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "a2bddab3-ee8f-4e22-b407-beba508677b0", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d-kserve-route-authn"}
2026-07-13T03:38:06Z	DEBUG	Triggering LLMInferenceService Cleanup for Namespace	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "a2bddab3-ee8f-4e22-b407-beba508677b0", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:38:06Z	DEBUG	Cleaning up LLMInferenceService sub-resources	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "a2bddab3-ee8f-4e22-b407-beba508677b0", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:38:06Z	INFO	Stop LLMInferenceService reconciliation	{"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-766a483d","namespace":"ai-tenant-e2e-mt-766a483d"}, "namespace": "ai-tenant-e2e-mt-766a483d", "name": "e2e-lifecycle-model-766a483d", "reconcileID": "481c9521-bc73-4216-88ab-dc1479b9e31e", "LLMInferenceService": "e2e-lifecycle-model-766a483d", "namespace": "ai-tenant-e2e-mt-766a483d"}
2026-07-13T03:38:06Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "e5ca0859-27bd-4526-802e-82d162b3dc84", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:38:06Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "e5ca0859-27bd-4526-802e-82d162b3dc84", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress", "name": "e2e-mt-766a483d-authn-ssl"}
2026-07-13T03:38:06Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-766a483d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-766a483d", "reconcileID": "e5ca0859-27bd-4526-802e-82d162b3dc84", "Gateway": "e2e-mt-766a483d", "namespace": "openshift-ingress"}
2026-07-13T03:38:15Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-1b2c7e07"}, "namespace": "ai-tenant-e2e-mt-1b2c7e07", "name": "openshift-service-ca.crt", "reconcileID": "2e27c202-aa95-42f4-b4f2-70998de9144e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-1b2c7e07"}
2026-07-13T03:38:15Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-1b2c7e07"}, "namespace": "ai-tenant-e2e-mt-1b2c7e07", "name": "odh-trusted-ca-bundle", "reconcileID": "76d5fad8-e6d1-40d4-be4a-60dac015dc7d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-1b2c7e07"}
2026-07-13T03:38:15Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-1b2c7e07"}, "namespace": "ai-tenant-e2e-mt-1b2c7e07", "name": "openshift-service-ca.crt", "reconcileID": "0fa43cef-36e1-47d5-8e60-36b0ae70fc2f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-1b2c7e07", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:38:15Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-1b2c7e07"}, "namespace": "ai-tenant-e2e-mt-1b2c7e07", "name": "openshift-service-ca.crt", "reconcileID": "0fa43cef-36e1-47d5-8e60-36b0ae70fc2f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-1b2c7e07"}
2026-07-13T03:38:15Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-1b2c7e07"}, "namespace": "ai-tenant-e2e-mt-1b2c7e07", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "857218df-7bc1-4325-89f9-ade8cfe10938", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-1b2c7e07"}
2026-07-13T03:38:15Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-1b2c7e07"}, "namespace": "ai-tenant-e2e-mt-1b2c7e07", "name": "openshift-service-ca.crt", "reconcileID": "d7c85e55-6ae8-4cfd-9636-4d01abd3e7ba", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-1b2c7e07"}
2026-07-13T03:38:15Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-1b2c7e07"}, "namespace": "ai-tenant-e2e-mt-1b2c7e07", "name": "odh-trusted-ca-bundle", "reconcileID": "8644872b-4aad-48e3-87fc-45846fc3b3ae", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-1b2c7e07", "update": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:38:15Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-1b2c7e07"}, "namespace": "ai-tenant-e2e-mt-1b2c7e07", "name": "odh-trusted-ca-bundle", "reconcileID": "8644872b-4aad-48e3-87fc-45846fc3b3ae", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-1b2c7e07"}
2026-07-13T03:38:15Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-1b2c7e07"}, "namespace": "ai-tenant-e2e-mt-1b2c7e07", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5067ebc9-333d-43ec-90f0-b5c1c15ddc21", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-1b2c7e07"}
2026-07-13T03:38:16Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-651f3984"}, "namespace": "ai-tenant-e2e-mt-651f3984", "name": "openshift-service-ca.crt", "reconcileID": "48bbca53-33f6-4feb-8ed1-a6a1c53c9f60", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-651f3984"}
2026-07-13T03:38:16Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-651f3984"}, "namespace": "ai-tenant-e2e-mt-651f3984", "name": "odh-trusted-ca-bundle", "reconcileID": "6d8823a0-2749-452e-bbf0-ee6d78e2ca24", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-651f3984"}
2026-07-13T03:38:16Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-651f3984"}, "namespace": "ai-tenant-e2e-mt-651f3984", "name": "openshift-service-ca.crt", "reconcileID": "a7134afa-5d53-4c7e-9dca-f244999898e4", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-651f3984", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:38:16Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-651f3984"}, "namespace": "ai-tenant-e2e-mt-651f3984", "name": "openshift-service-ca.crt", "reconcileID": "a7134afa-5d53-4c7e-9dca-f244999898e4", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-651f3984"}
2026-07-13T03:38:16Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-651f3984"}, "namespace": "ai-tenant-e2e-mt-651f3984", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "657cbbe8-bd32-4e92-a26d-84daf8bf617b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-651f3984"}
2026-07-13T03:38:16Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-651f3984"}, "namespace": "ai-tenant-e2e-mt-651f3984", "name": "openshift-service-ca.crt", "reconcileID": "5f3aa178-0e20-4125-bb0f-560c7092bccf", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-651f3984"}
2026-07-13T03:38:16Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-651f3984"}, "namespace": "ai-tenant-e2e-mt-651f3984", "name": "odh-trusted-ca-bundle", "reconcileID": "70778926-bf21-4a50-b5ef-ba6ddbd60c13", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-651f3984", "update": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:38:16Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-651f3984"}, "namespace": "ai-tenant-e2e-mt-651f3984", "name": "odh-trusted-ca-bundle", "reconcileID": "70778926-bf21-4a50-b5ef-ba6ddbd60c13", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-651f3984"}
2026-07-13T03:38:16Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-651f3984"}, "namespace": "ai-tenant-e2e-mt-651f3984", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "706e3f57-b722-45bd-b187-9866f17ca248", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-651f3984"}
2026-07-13T03:38:23Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-1b2c7e07"}, "namespace": "ai-tenant-e2e-mt-1b2c7e07", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f097e25b-5cb6-4993-9adb-9dc0aa6acfe2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-1b2c7e07"}
2026-07-13T03:38:23Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-1b2c7e07"}, "namespace": "ai-tenant-e2e-mt-1b2c7e07", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f097e25b-5cb6-4993-9adb-9dc0aa6acfe2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-1b2c7e07", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:38:23Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-1b2c7e07"}, "namespace": "ai-tenant-e2e-mt-1b2c7e07", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f097e25b-5cb6-4993-9adb-9dc0aa6acfe2", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-1b2c7e07 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-13T03:38:23Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-1b2c7e07"}, "namespace": "ai-tenant-e2e-mt-1b2c7e07", "name": "odh-trusted-ca-bundle", "reconcileID": "a4c5cccf-7888-4826-97ae-3f7dc7e7fb75", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-1b2c7e07"}
2026-07-13T03:38:23Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-1b2c7e07"}, "namespace": "ai-tenant-e2e-mt-1b2c7e07", "name": "odh-trusted-ca-bundle", "reconcileID": "a4c5cccf-7888-4826-97ae-3f7dc7e7fb75", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-1b2c7e07"}
2026-07-13T03:38:23Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-1b2c7e07"}, "namespace": "ai-tenant-e2e-mt-1b2c7e07", "name": "openshift-service-ca.crt", "reconcileID": "e32b3085-6009-41dd-84f2-fe6729af7537", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-1b2c7e07"}
2026-07-13T03:38:23Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-1b2c7e07"}, "namespace": "ai-tenant-e2e-mt-1b2c7e07", "name": "openshift-service-ca.crt", "reconcileID": "e32b3085-6009-41dd-84f2-fe6729af7537", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-1b2c7e07"}
2026-07-13T03:38:23Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-1b2c7e07"}, "namespace": "ai-tenant-e2e-mt-1b2c7e07", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "932454fa-5e99-47df-95bc-db41e9b4d282", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-1b2c7e07"}
2026-07-13T03:38:23Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-1b2c7e07"}, "namespace": "ai-tenant-e2e-mt-1b2c7e07", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "932454fa-5e99-47df-95bc-db41e9b4d282", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-1b2c7e07"}
2026-07-13T03:38:29Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-651f3984"}, "namespace": "ai-tenant-e2e-mt-651f3984", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "85d6fbd5-bce6-4c1e-b12b-a0ed1b6c20a0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-651f3984"}
2026-07-13T03:38:29Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-651f3984"}, "namespace": "ai-tenant-e2e-mt-651f3984", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "85d6fbd5-bce6-4c1e-b12b-a0ed1b6c20a0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-651f3984", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:38:29Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-651f3984"}, "namespace": "ai-tenant-e2e-mt-651f3984", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "85d6fbd5-bce6-4c1e-b12b-a0ed1b6c20a0", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-651f3984 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-13T03:38:29Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-651f3984"}, "namespace": "ai-tenant-e2e-mt-651f3984", "name": "odh-trusted-ca-bundle", "reconcileID": "1f950e9c-074b-4348-9ccf-6b853cb7805e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-651f3984"}
2026-07-13T03:38:29Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-651f3984"}, "namespace": "ai-tenant-e2e-mt-651f3984", "name": "odh-trusted-ca-bundle", "reconcileID": "1f950e9c-074b-4348-9ccf-6b853cb7805e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-651f3984"}
2026-07-13T03:38:29Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-651f3984"}, "namespace": "ai-tenant-e2e-mt-651f3984", "name": "openshift-service-ca.crt", "reconcileID": "6c2fe43c-0dab-4b04-a01a-b1f793b171ba", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-651f3984"}
2026-07-13T03:38:29Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-651f3984"}, "namespace": "ai-tenant-e2e-mt-651f3984", "name": "openshift-service-ca.crt", "reconcileID": "6c2fe43c-0dab-4b04-a01a-b1f793b171ba", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-651f3984"}
2026-07-13T03:38:29Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-651f3984"}, "namespace": "ai-tenant-e2e-mt-651f3984", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1bc7085e-be94-4bac-baac-863a69e24f30", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-651f3984"}
2026-07-13T03:38:29Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-651f3984"}, "namespace": "ai-tenant-e2e-mt-651f3984", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1bc7085e-be94-4bac-baac-863a69e24f30", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-651f3984"}
2026-07-13T03:38:29Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-8ff9d671"}, "namespace": "ai-tenant-e2e-mt-8ff9d671", "name": "openshift-service-ca.crt", "reconcileID": "52b4332b-d5a3-45f1-9617-62c43172dbbc", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-8ff9d671"}
2026-07-13T03:38:29Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-8ff9d671"}, "namespace": "ai-tenant-e2e-mt-8ff9d671", "name": "odh-trusted-ca-bundle", "reconcileID": "0dc84bd1-ea2b-4f8a-86b6-df9f3003e10d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-8ff9d671"}
2026-07-13T03:38:29Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-8ff9d671"}, "namespace": "ai-tenant-e2e-mt-8ff9d671", "name": "openshift-service-ca.crt", "reconcileID": "46c81d09-ce31-434a-a356-2445d101563f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-8ff9d671", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:38:29Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-8ff9d671"}, "namespace": "ai-tenant-e2e-mt-8ff9d671", "name": "openshift-service-ca.crt", "reconcileID": "46c81d09-ce31-434a-a356-2445d101563f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-8ff9d671"}
2026-07-13T03:38:29Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-8ff9d671"}, "namespace": "ai-tenant-e2e-mt-8ff9d671", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3b3292b8-ea9d-4229-a6b3-43c0b27a2d70", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-8ff9d671"}
2026-07-13T03:38:29Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-8ff9d671"}, "namespace": "ai-tenant-e2e-mt-8ff9d671", "name": "openshift-service-ca.crt", "reconcileID": "dabee98c-e9b0-489f-8b07-edb54b01ba13", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-8ff9d671"}
2026-07-13T03:38:30Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-8ff9d671"}, "namespace": "ai-tenant-e2e-mt-8ff9d671", "name": "odh-trusted-ca-bundle", "reconcileID": "900f74d6-3548-4fae-a396-6ddb18b42af7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-8ff9d671", "update": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:38:30Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-8ff9d671"}, "namespace": "ai-tenant-e2e-mt-8ff9d671", "name": "odh-trusted-ca-bundle", "reconcileID": "900f74d6-3548-4fae-a396-6ddb18b42af7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-8ff9d671"}
2026-07-13T03:38:30Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-8ff9d671"}, "namespace": "ai-tenant-e2e-mt-8ff9d671", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "64e1b6e5-69f6-4f59-8039-e1f075f7d1e5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-8ff9d671"}
2026-07-13T03:39:07Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-8ff9d671"}, "namespace": "ai-tenant-e2e-mt-8ff9d671", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a1e03936-5e36-4303-b728-b45fd5552902", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-8ff9d671"}
2026-07-13T03:39:07Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-8ff9d671"}, "namespace": "ai-tenant-e2e-mt-8ff9d671", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a1e03936-5e36-4303-b728-b45fd5552902", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-8ff9d671", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:39:07Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-8ff9d671"}, "namespace": "ai-tenant-e2e-mt-8ff9d671", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a1e03936-5e36-4303-b728-b45fd5552902", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-8ff9d671 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-13T03:39:07Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-8ff9d671"}, "namespace": "ai-tenant-e2e-mt-8ff9d671", "name": "odh-trusted-ca-bundle", "reconcileID": "99e7e7d7-5bbf-4bbd-8b2d-ce2e5163f491", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-8ff9d671"}
2026-07-13T03:39:07Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-8ff9d671"}, "namespace": "ai-tenant-e2e-mt-8ff9d671", "name": "odh-trusted-ca-bundle", "reconcileID": "99e7e7d7-5bbf-4bbd-8b2d-ce2e5163f491", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-8ff9d671"}
2026-07-13T03:39:07Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-8ff9d671"}, "namespace": "ai-tenant-e2e-mt-8ff9d671", "name": "openshift-service-ca.crt", "reconcileID": "e6199bcf-6502-4d0f-b75d-928bf14493aa", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-8ff9d671"}
2026-07-13T03:39:07Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-8ff9d671"}, "namespace": "ai-tenant-e2e-mt-8ff9d671", "name": "openshift-service-ca.crt", "reconcileID": "e6199bcf-6502-4d0f-b75d-928bf14493aa", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-8ff9d671"}
2026-07-13T03:39:07Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-8ff9d671"}, "namespace": "ai-tenant-e2e-mt-8ff9d671", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dbdfcfb0-c785-4b90-8b74-1331ee1bae46", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-8ff9d671"}
2026-07-13T03:39:07Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-8ff9d671"}, "namespace": "ai-tenant-e2e-mt-8ff9d671", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dbdfcfb0-c785-4b90-8b74-1331ee1bae46", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-8ff9d671"}
2026-07-13T03:39:07Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-3eb586","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586", "reconcileID": "c682f3e7-ea61-4dbd-ab54-8e9f19f7a5a9", "Gateway": "e2e-inf-a-3eb586", "namespace": "openshift-ingress"}
2026-07-13T03:39:07Z	INFO	Creating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-3eb586","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586", "reconcileID": "c682f3e7-ea61-4dbd-ab54-8e9f19f7a5a9", "Gateway": "e2e-inf-a-3eb586", "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586-authn-ssl"}
2026-07-13T03:39:07Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-3eb586","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586", "reconcileID": "c682f3e7-ea61-4dbd-ab54-8e9f19f7a5a9", "Gateway": "e2e-inf-a-3eb586", "namespace": "openshift-ingress"}
2026-07-13T03:39:07Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-3eb586","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586", "reconcileID": "94e6073a-c265-4ff4-8571-9b49f11b27ec", "Gateway": "e2e-inf-a-3eb586", "namespace": "openshift-ingress"}
2026-07-13T03:39:07Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-3eb586","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586", "reconcileID": "94e6073a-c265-4ff4-8571-9b49f11b27ec", "Gateway": "e2e-inf-a-3eb586", "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586-authn-ssl"}
2026-07-13T03:39:07Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-3eb586","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586", "reconcileID": "94e6073a-c265-4ff4-8571-9b49f11b27ec", "Gateway": "e2e-inf-a-3eb586", "namespace": "openshift-ingress"}
2026-07-13T03:39:07Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-3eb586","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586", "reconcileID": "7e993d50-fc27-4c8f-966a-038aa5949822", "Gateway": "e2e-inf-a-3eb586", "namespace": "openshift-ingress"}
2026-07-13T03:39:07Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-3eb586","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586", "reconcileID": "7e993d50-fc27-4c8f-966a-038aa5949822", "Gateway": "e2e-inf-a-3eb586", "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586-authn-ssl"}
2026-07-13T03:39:07Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-3eb586","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586", "reconcileID": "7e993d50-fc27-4c8f-966a-038aa5949822", "Gateway": "e2e-inf-a-3eb586", "namespace": "openshift-ingress"}
2026-07-13T03:39:07Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-3eb586","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586", "reconcileID": "75e794e4-c958-4d97-aa4f-5f5cd7111dab", "Gateway": "e2e-inf-a-3eb586", "namespace": "openshift-ingress"}
2026-07-13T03:39:07Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-3eb586","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586", "reconcileID": "75e794e4-c958-4d97-aa4f-5f5cd7111dab", "Gateway": "e2e-inf-a-3eb586", "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586-authn-ssl"}
2026-07-13T03:39:07Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-3eb586","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586", "reconcileID": "75e794e4-c958-4d97-aa4f-5f5cd7111dab", "Gateway": "e2e-inf-a-3eb586", "namespace": "openshift-ingress"}
2026-07-13T03:39:07Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-3eb586","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586", "reconcileID": "6d39f48e-26a2-4829-a777-fdad04723a82", "Gateway": "e2e-inf-a-3eb586", "namespace": "openshift-ingress"}
2026-07-13T03:39:07Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-3eb586","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586", "reconcileID": "6d39f48e-26a2-4829-a777-fdad04723a82", "Gateway": "e2e-inf-a-3eb586", "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586-authn-ssl"}
2026-07-13T03:39:08Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-3eb586","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586", "reconcileID": "6d39f48e-26a2-4829-a777-fdad04723a82", "Gateway": "e2e-inf-a-3eb586", "namespace": "openshift-ingress"}
2026-07-13T03:39:08Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-3eb586","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586", "reconcileID": "fd31eb5e-8668-4fe3-82c5-ab00a7388d15", "Gateway": "e2e-inf-a-3eb586", "namespace": "openshift-ingress"}
2026-07-13T03:39:08Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-3eb586","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586", "reconcileID": "fd31eb5e-8668-4fe3-82c5-ab00a7388d15", "Gateway": "e2e-inf-a-3eb586", "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586-authn-ssl"}
2026-07-13T03:39:08Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-3eb586","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586", "reconcileID": "fd31eb5e-8668-4fe3-82c5-ab00a7388d15", "Gateway": "e2e-inf-a-3eb586", "namespace": "openshift-ingress"}
2026-07-13T03:39:09Z	DEBUG	Reconciling EnvoyFilter for Gateway	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-3eb586","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586", "reconcileID": "53cfab7f-9922-4371-acec-1e198b66345c", "Gateway": "e2e-inf-a-3eb586", "namespace": "openshift-ingress"}
2026-07-13T03:39:09Z	INFO	Updating EnvoyFilter	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-3eb586","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586", "reconcileID": "53cfab7f-9922-4371-acec-1e198b66345c", "Gateway": "e2e-inf-a-3eb586", "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586-authn-ssl"}
2026-07-13T03:39:09Z	DEBUG	Successfully reconciled Gateway resources	{"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-3eb586","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-3eb586", "reconcileID": "53cfab7f-9922-4371-acec-1e198b66345c", "Gateway": "e2e-inf-a-3eb586", "namespace": "openshift-ingress"}
2026-07-13T03:39:10Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-3eb586"}, "namespace": "ai-tenant-e2e-inf-a-3eb586", "name": "openshift-service-ca.crt", "reconcileID": "4944d1e0-1908-4667-a88a-9ebb31dffa8b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-3eb586"}
2026-07-13T03:39:10Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-3eb586"}, "namespace": "ai-tenant-e2e-inf-a-3eb586", "name": "odh-trusted-ca-bundle", "reconcileID": "fe0a8fa7-44da-4c1b-b006-76cb45cb4843", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-3eb586"}
2026-07-13T03:39:10Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-3eb586"}, "namespace": "ai-tenant-e2e-inf-a-3eb586", "name": "openshift-service-ca.crt", "reconcileID": "6d06e3c1-99e8-4cd2-9204-739f10d29021", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-3eb586", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:39:10Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-3eb586"}, "namespace": "ai-tenant-e2e-inf-a-3eb586", "name": "openshift-service-ca.crt", "reconcileID": "6d06e3c1-99e8-4cd2-9204-739f10d29021", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-3eb586"}
2026-07-13T03:39:10Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-3eb586"}, "namespace": "ai-tenant-e2e-inf-a-3eb586", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dcbc31bd-bbd0-4910-ae48-18d947072a02", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-3eb586"}
2026-07-13T03:39:10Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-3eb586"}, "namespace": "ai-tenant-e2e-inf-a-3eb586", "name": "openshift-service-ca.crt", "reconcileID": "9ab52bfc-8b6e-4415-8b99-713798b9002b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-3eb586"}
2026-07-13T03:39:10Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-3eb586"}, "namespace": "ai-tenant-e2e-inf-a-3eb586", "name": "odh-trusted-ca-bundle", "reconcileID": "e7d8ce5f-4303-4eea-a168-cbc4c437d8a8", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-3eb586", "update": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:39:10Z	DEBUG	Deleted the storage-config Secret to update the custom cert	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-3eb586"}, "namespace": "ai-tenant-e2e-inf-a-3eb586", "name": "odh-trusted-ca-bundle", "reconcileID": "e7d8ce5f-4303-4eea-a168-cbc4c437d8a8", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-3eb586"}
2026-07-13T03:39:10Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-3eb586"}, "namespace": "ai-tenant-e2e-inf-a-3eb586", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a2c66a45-85ad-47b3-8c27-58a90309a64d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-3eb586"}
2026-07-13T03:42:25Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-3eb586"}, "namespace": "ai-tenant-e2e-inf-a-3eb586", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b0654bf2-e9d4-4cf0-a403-635aeac0f0c0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-3eb586"}
2026-07-13T03:42:25Z	DEBUG	Delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-3eb586"}, "namespace": "ai-tenant-e2e-inf-a-3eb586", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b0654bf2-e9d4-4cf0-a403-635aeac0f0c0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-3eb586", "create": "odh-kserve-custom-ca-bundle"}
2026-07-13T03:42:25Z	ERROR	Reconciler error	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-3eb586"}, "namespace": "ai-tenant-e2e-inf-a-3eb586", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b0654bf2-e9d4-4cf0-a403-635aeac0f0c0", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-inf-a-3eb586 because it is being terminated"}
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263
sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2
	/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224
2026-07-13T03:42:25Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-3eb586"}, "namespace": "ai-tenant-e2e-inf-a-3eb586", "name": "odh-trusted-ca-bundle", "reconcileID": "1a0228e2-3be7-42e0-ab5f-d709edb81cad", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-3eb586"}
2026-07-13T03:42:25Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-3eb586"}, "namespace": "ai-tenant-e2e-inf-a-3eb586", "name": "odh-trusted-ca-bundle", "reconcileID": "1a0228e2-3be7-42e0-ab5f-d709edb81cad", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-3eb586"}
2026-07-13T03:42:25Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-3eb586"}, "namespace": "ai-tenant-e2e-inf-a-3eb586", "name": "openshift-service-ca.crt", "reconcileID": "66729610-60a9-4d54-ba48-13cd06f54e19", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-3eb586"}
2026-07-13T03:42:25Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-3eb586"}, "namespace": "ai-tenant-e2e-inf-a-3eb586", "name": "openshift-service-ca.crt", "reconcileID": "66729610-60a9-4d54-ba48-13cd06f54e19", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-3eb586"}
2026-07-13T03:42:25Z	INFO	Cert ConfigMap not found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-3eb586"}, "namespace": "ai-tenant-e2e-inf-a-3eb586", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "103ec84d-2c68-42e6-87c1-c837f2935068", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-3eb586"}
2026-07-13T03:42:25Z	DEBUG	No delta found	{"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-3eb586"}, "namespace": "ai-tenant-e2e-inf-a-3eb586", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "103ec84d-2c68-42e6-87c1-c837f2935068", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-3eb586"}