2026-07-09T03:10:44Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-3c375e97"}, "namespace": "e2e-webhook-auth-3c375e97", "name": "openshift-service-ca.crt", "reconcileID": "f3664f55-1258-4d3e-8cbf-7ebec3ca500e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-3c375e97"} 2026-07-09T03:10:44Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-3c375e97"}, "namespace": "e2e-webhook-auth-3c375e97", "name": "odh-trusted-ca-bundle", "reconcileID": "7c948990-d706-469f-ab22-e63601aa19d1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-3c375e97"} 2026-07-09T03:10:44Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-3c375e97"}, "namespace": "e2e-webhook-auth-3c375e97", "name": "openshift-service-ca.crt", "reconcileID": "494096e5-868c-43a2-b072-3441fbebf067", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-3c375e97", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:10:44Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-3c375e97"}, "namespace": "e2e-webhook-auth-3c375e97", "name": "openshift-service-ca.crt", "reconcileID": "494096e5-868c-43a2-b072-3441fbebf067", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-3c375e97"} 2026-07-09T03:10:44Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-3c375e97"}, "namespace": "e2e-webhook-auth-3c375e97", "name": "openshift-service-ca.crt", "reconcileID": "bf83f8ad-0b3d-4603-b4a2-ee79e20fde69", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-3c375e97"} 2026-07-09T03:10:44Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-3c375e97"}, "namespace": "e2e-webhook-auth-3c375e97", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "49258524-25c1-454e-8bb0-e445ddcfb8a4", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-3c375e97"} 2026-07-09T03:10:44Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-3c375e97"}, "namespace": "e2e-webhook-auth-3c375e97", "name": "odh-trusted-ca-bundle", "reconcileID": "8c9d929c-08d6-41d6-bb5e-5666644c509a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-3c375e97", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:10:44Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-3c375e97"}, "namespace": "e2e-webhook-auth-3c375e97", "name": "odh-trusted-ca-bundle", "reconcileID": "8c9d929c-08d6-41d6-bb5e-5666644c509a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-3c375e97"} 2026-07-09T03:10:44Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-3c375e97"}, "namespace": "e2e-webhook-auth-3c375e97", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "08117086-f85b-4bf8-9598-47e950612fde", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-3c375e97"} 2026-07-09T03:10:50Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-3c375e97"}, "namespace": "e2e-webhook-auth-3c375e97", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9003b807-6263-45f2-9526-39cbce1cff15", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-3c375e97"} 2026-07-09T03:10:50Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-3c375e97"}, "namespace": "e2e-webhook-auth-3c375e97", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9003b807-6263-45f2-9526-39cbce1cff15", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-3c375e97", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:10:50Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-3c375e97"}, "namespace": "e2e-webhook-auth-3c375e97", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9003b807-6263-45f2-9526-39cbce1cff15", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-auth-3c375e97 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T03:10:50Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-3c375e97"}, "namespace": "e2e-webhook-auth-3c375e97", "name": "odh-trusted-ca-bundle", "reconcileID": "c5bb5687-4033-47ed-9250-413ff7b9cb4a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-3c375e97"} 2026-07-09T03:10:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-3c375e97"}, "namespace": "e2e-webhook-auth-3c375e97", "name": "odh-trusted-ca-bundle", "reconcileID": "c5bb5687-4033-47ed-9250-413ff7b9cb4a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-3c375e97"} 2026-07-09T03:10:50Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-3c375e97"}, "namespace": "e2e-webhook-auth-3c375e97", "name": "openshift-service-ca.crt", "reconcileID": "250d565d-a52b-445e-81d9-15b49a74a85a", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-3c375e97"} 2026-07-09T03:10:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-3c375e97"}, "namespace": "e2e-webhook-auth-3c375e97", "name": "openshift-service-ca.crt", "reconcileID": "250d565d-a52b-445e-81d9-15b49a74a85a", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-3c375e97"} 2026-07-09T03:10:50Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-3c375e97"}, "namespace": "e2e-webhook-auth-3c375e97", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ca4327f4-1e11-434f-a356-774b34639c9d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-3c375e97"} 2026-07-09T03:10:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-3c375e97"}, "namespace": "e2e-webhook-auth-3c375e97", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ca4327f4-1e11-434f-a356-774b34639c9d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-3c375e97"} 2026-07-09T03:10:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "c1d36fa0-59fd-442a-9975-344e9b9a6ab5", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:10:57Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "c1d36fa0-59fd-442a-9975-344e9b9a6ab5", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress", "name": "e2e-mt-28444007-authn-ssl"} 2026-07-09T03:10:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "c1d36fa0-59fd-442a-9975-344e9b9a6ab5", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:10:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "a0a06b1d-c80b-4d31-bb60-b5227869d26b", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:10:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "a0a06b1d-c80b-4d31-bb60-b5227869d26b", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress", "name": "e2e-mt-28444007-authn-ssl"} 2026-07-09T03:10:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "a0a06b1d-c80b-4d31-bb60-b5227869d26b", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:10:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "51cb7cf8-f1ef-40a0-a5b8-1ea605d64be7", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:10:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "51cb7cf8-f1ef-40a0-a5b8-1ea605d64be7", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress", "name": "e2e-mt-28444007-authn-ssl"} 2026-07-09T03:10:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "51cb7cf8-f1ef-40a0-a5b8-1ea605d64be7", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:10:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "2060441d-808e-4de1-a962-ef62fc87bd1f", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:10:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "2060441d-808e-4de1-a962-ef62fc87bd1f", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress", "name": "e2e-mt-28444007-authn-ssl"} 2026-07-09T03:10:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "2060441d-808e-4de1-a962-ef62fc87bd1f", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:10:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "bc04cfc5-59e7-40ec-8fda-5c15f0cba48f", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:10:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "bc04cfc5-59e7-40ec-8fda-5c15f0cba48f", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress", "name": "e2e-mt-28444007-authn-ssl"} 2026-07-09T03:10:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "bc04cfc5-59e7-40ec-8fda-5c15f0cba48f", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:10:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "e411e11a-e836-4448-80ac-8ad2eb8c52b5", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:10:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "e411e11a-e836-4448-80ac-8ad2eb8c52b5", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress", "name": "e2e-mt-28444007-authn-ssl"} 2026-07-09T03:10:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "e411e11a-e836-4448-80ac-8ad2eb8c52b5", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:10:58Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "c60b6aa3-a181-4c08-8a58-dfa591d91f81", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:10:58Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "c60b6aa3-a181-4c08-8a58-dfa591d91f81", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress", "name": "e2e-mt-28444007-authn-ssl"} 2026-07-09T03:10:58Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "c60b6aa3-a181-4c08-8a58-dfa591d91f81", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:11:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "openshift-service-ca.crt", "reconcileID": "3e898c4c-bbe1-496b-84fd-554f685e445a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "odh-trusted-ca-bundle", "reconcileID": "c9b2b40b-745d-4748-920c-859bc7d02e2b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:00Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "openshift-service-ca.crt", "reconcileID": "e8d23da0-f540-4ff3-ab04-ed1dccf25623", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-28444007", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:11:00Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "openshift-service-ca.crt", "reconcileID": "e8d23da0-f540-4ff3-ab04-ed1dccf25623", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "openshift-service-ca.crt", "reconcileID": "ecabbe97-fb09-4041-89db-4e20a0f33b31", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1e508c41-2641-4e59-bc49-561a2e54a264", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:00Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "odh-trusted-ca-bundle", "reconcileID": "63af45e4-8534-4f5e-ac94-a1930bacbd11", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-28444007", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:11:00Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "odh-trusted-ca-bundle", "reconcileID": "63af45e4-8534-4f5e-ac94-a1930bacbd11", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "64e60e34-3c2d-40c7-a5a4-cb92ef8f5953", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:01Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "fed459ca-57a4-4097-be5b-137f54059520", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:11:01Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "fed459ca-57a4-4097-be5b-137f54059520", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress", "name": "e2e-mt-28444007-authn-ssl"} 2026-07-09T03:11:01Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "fed459ca-57a4-4097-be5b-137f54059520", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:11:01Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "79fae1cf-ec27-42ff-98f4-76adb135e483", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-09T03:11:05Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "f55fe010-9673-49e0-a791-21a7b07f8c81", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:05Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "f55fe010-9673-49e0-a791-21a7b07f8c81", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:05Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "f55fe010-9673-49e0-a791-21a7b07f8c81", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:05Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "11418f66-1442-438a-a5a8-525d5a4fd206", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:11:05Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "f55fe010-9673-49e0-a791-21a7b07f8c81", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007-kserve-route-authn"} 2026-07-09T03:11:05Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "f55fe010-9673-49e0-a791-21a7b07f8c81", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007", "authType": "userdefined"} 2026-07-09T03:11:05Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "f55fe010-9673-49e0-a791-21a7b07f8c81", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:05Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "11418f66-1442-438a-a5a8-525d5a4fd206", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress", "name": "e2e-mt-28444007-authn-ssl"} 2026-07-09T03:11:05Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "11418f66-1442-438a-a5a8-525d5a4fd206", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:11:05Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "d47535bf-1848-43b4-bde3-35d652ba295b", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:05Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "d47535bf-1848-43b4-bde3-35d652ba295b", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:05Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "d47535bf-1848-43b4-bde3-35d652ba295b", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:05Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "d47535bf-1848-43b4-bde3-35d652ba295b", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007-kserve-route-authn"} 2026-07-09T03:11:05Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "d47535bf-1848-43b4-bde3-35d652ba295b", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007", "authType": "userdefined"} 2026-07-09T03:11:05Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "d47535bf-1848-43b4-bde3-35d652ba295b", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:06Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "a0ba1199-294f-4f25-a273-67c759406150", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:11:06Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "a0ba1199-294f-4f25-a273-67c759406150", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress", "name": "e2e-mt-28444007-authn-ssl"} 2026-07-09T03:11:06Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "a0ba1199-294f-4f25-a273-67c759406150", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:11:08Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "59631d54-9aef-4051-8fc7-0ef30ac69916", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:08Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "59631d54-9aef-4051-8fc7-0ef30ac69916", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:08Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "59631d54-9aef-4051-8fc7-0ef30ac69916", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:08Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "59631d54-9aef-4051-8fc7-0ef30ac69916", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007-kserve-route-authn"} 2026-07-09T03:11:08Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "59631d54-9aef-4051-8fc7-0ef30ac69916", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007", "authType": "userdefined"} 2026-07-09T03:11:08Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "59631d54-9aef-4051-8fc7-0ef30ac69916", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:12Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "7bc2f5bc-6636-4078-abbf-53f4d383b9ad", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:11:12Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "7bc2f5bc-6636-4078-abbf-53f4d383b9ad", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress", "name": "e2e-mt-28444007-authn-ssl"} 2026-07-09T03:11:12Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "7bc2f5bc-6636-4078-abbf-53f4d383b9ad", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:11:41Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "af42b8ce-4017-4de2-8dfd-d473a121d2ed", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:41Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "af42b8ce-4017-4de2-8dfd-d473a121d2ed", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:41Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "af42b8ce-4017-4de2-8dfd-d473a121d2ed", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:41Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "af42b8ce-4017-4de2-8dfd-d473a121d2ed", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007-kserve-route-authn"} 2026-07-09T03:11:41Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "af42b8ce-4017-4de2-8dfd-d473a121d2ed", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007", "authType": "userdefined"} 2026-07-09T03:11:41Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "af42b8ce-4017-4de2-8dfd-d473a121d2ed", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:46Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "8434a64f-5f44-478c-884f-a7b3f1a1b516", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:11:46Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "8434a64f-5f44-478c-884f-a7b3f1a1b516", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress", "name": "e2e-mt-28444007-authn-ssl"} 2026-07-09T03:11:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "8434a64f-5f44-478c-884f-a7b3f1a1b516", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:11:47Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "db3daba2-706f-4e66-a5ae-4060d218d6f4", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:11:47Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "db3daba2-706f-4e66-a5ae-4060d218d6f4", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress", "name": "e2e-mt-28444007-authn-ssl"} 2026-07-09T03:11:47Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "db3daba2-706f-4e66-a5ae-4060d218d6f4", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:11:47Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "708ace89-75f3-4537-9e59-655cf802e6e4", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:11:47Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "708ace89-75f3-4537-9e59-655cf802e6e4", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress", "name": "e2e-mt-28444007-authn-ssl"} 2026-07-09T03:11:47Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "708ace89-75f3-4537-9e59-655cf802e6e4", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:11:54Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "2aabe5da-69ba-425e-84af-bcc8d611b083", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:54Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "2aabe5da-69ba-425e-84af-bcc8d611b083", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:54Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "2aabe5da-69ba-425e-84af-bcc8d611b083", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:11:54Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "2aabe5da-69ba-425e-84af-bcc8d611b083", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007-kserve-route-authn"} 2026-07-09T03:11:54Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "2aabe5da-69ba-425e-84af-bcc8d611b083", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007", "authType": "userdefined"} 2026-07-09T03:11:54Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "2aabe5da-69ba-425e-84af-bcc8d611b083", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:12:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a1ce1c2e-c805-4eab-8651-c77620a7b751", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:12:29Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a1ce1c2e-c805-4eab-8651-c77620a7b751", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-28444007", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:12:29Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a1ce1c2e-c805-4eab-8651-c77620a7b751", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-28444007 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T03:12:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "odh-trusted-ca-bundle", "reconcileID": "0b8a5b5f-ab32-4a91-a30b-4ea89a039d5d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:12:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "odh-trusted-ca-bundle", "reconcileID": "0b8a5b5f-ab32-4a91-a30b-4ea89a039d5d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:12:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "openshift-service-ca.crt", "reconcileID": "1ab09acc-0f1b-4099-bc5a-ca6120c71e10", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:12:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "openshift-service-ca.crt", "reconcileID": "1ab09acc-0f1b-4099-bc5a-ca6120c71e10", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:12:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c1037ba6-2a8e-48c6-8505-276f9fd957fd", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:12:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c1037ba6-2a8e-48c6-8505-276f9fd957fd", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:12:29Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "6f789550-c24c-4401-9197-85e779e9c6ee", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:12:29Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "6f789550-c24c-4401-9197-85e779e9c6ee", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress", "name": "e2e-mt-28444007-authn-ssl"} 2026-07-09T03:12:29Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "6f789550-c24c-4401-9197-85e779e9c6ee", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:12:30Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "4661d3ea-8a82-47e7-8cbf-c198f7360fdd", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:12:30Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "4661d3ea-8a82-47e7-8cbf-c198f7360fdd", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007"} 2026-07-09T03:12:30Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "4661d3ea-8a82-47e7-8cbf-c198f7360fdd", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:12:30Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "4661d3ea-8a82-47e7-8cbf-c198f7360fdd", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:12:30Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "4661d3ea-8a82-47e7-8cbf-c198f7360fdd", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007-kserve-route-authn"} 2026-07-09T03:12:30Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "4661d3ea-8a82-47e7-8cbf-c198f7360fdd", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007-kserve-route-authn"} 2026-07-09T03:12:30Z DEBUG Triggering LLMInferenceService Cleanup for Namespace {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "4661d3ea-8a82-47e7-8cbf-c198f7360fdd", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:12:30Z DEBUG Cleaning up LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "4661d3ea-8a82-47e7-8cbf-c198f7360fdd", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:12:31Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "11f9ea40-119a-4bcf-bad9-8d31189927f2", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:12:31Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "11f9ea40-119a-4bcf-bad9-8d31189927f2", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007"} 2026-07-09T03:12:31Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "11f9ea40-119a-4bcf-bad9-8d31189927f2", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:12:31Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "11f9ea40-119a-4bcf-bad9-8d31189927f2", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:12:31Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "11f9ea40-119a-4bcf-bad9-8d31189927f2", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007-kserve-route-authn"} 2026-07-09T03:12:31Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "11f9ea40-119a-4bcf-bad9-8d31189927f2", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007-kserve-route-authn"} 2026-07-09T03:12:31Z DEBUG Triggering LLMInferenceService Cleanup for Namespace {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "11f9ea40-119a-4bcf-bad9-8d31189927f2", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:12:31Z DEBUG Cleaning up LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "11f9ea40-119a-4bcf-bad9-8d31189927f2", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:12:31Z INFO Stop LLMInferenceService reconciliation {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-28444007","namespace":"ai-tenant-e2e-mt-28444007"}, "namespace": "ai-tenant-e2e-mt-28444007", "name": "e2e-lifecycle-model-28444007", "reconcileID": "3f40d273-e7e6-452f-847a-3d8c18525c5c", "LLMInferenceService": "e2e-lifecycle-model-28444007", "namespace": "ai-tenant-e2e-mt-28444007"} 2026-07-09T03:12:31Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "5db8baca-51e3-4427-a98a-e7a17416c7c4", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:12:31Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "5db8baca-51e3-4427-a98a-e7a17416c7c4", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress", "name": "e2e-mt-28444007-authn-ssl"} 2026-07-09T03:12:31Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-28444007","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-28444007", "reconcileID": "5db8baca-51e3-4427-a98a-e7a17416c7c4", "Gateway": "e2e-mt-28444007", "namespace": "openshift-ingress"} 2026-07-09T03:12:37Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-8d991807"}, "namespace": "ai-tenant-e2e-mt-8d991807", "name": "openshift-service-ca.crt", "reconcileID": "724a001c-32de-4226-953a-f9fb1c584d3f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-8d991807"} 2026-07-09T03:12:37Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-8d991807"}, "namespace": "ai-tenant-e2e-mt-8d991807", "name": "odh-trusted-ca-bundle", "reconcileID": "ae5aa944-8170-4b47-9ec8-97e2b145efd8", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-8d991807"} 2026-07-09T03:12:37Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-8d991807"}, "namespace": "ai-tenant-e2e-mt-8d991807", "name": "openshift-service-ca.crt", "reconcileID": "a79b83fd-313b-4e9d-9149-2a38a3b6879c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-8d991807", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:12:37Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-8d991807"}, "namespace": "ai-tenant-e2e-mt-8d991807", "name": "openshift-service-ca.crt", "reconcileID": "a79b83fd-313b-4e9d-9149-2a38a3b6879c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-8d991807"} 2026-07-09T03:12:37Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-8d991807"}, "namespace": "ai-tenant-e2e-mt-8d991807", "name": "openshift-service-ca.crt", "reconcileID": "7ab42c46-61c9-4427-88f5-77324c556e97", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-8d991807"} 2026-07-09T03:12:37Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-8d991807"}, "namespace": "ai-tenant-e2e-mt-8d991807", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9b388b52-348e-49a6-bfde-37639e106623", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-8d991807"} 2026-07-09T03:12:37Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-8d991807"}, "namespace": "ai-tenant-e2e-mt-8d991807", "name": "odh-trusted-ca-bundle", "reconcileID": "72e9d64e-feae-449e-8ace-8e27242719b5", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-8d991807", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:12:37Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-8d991807"}, "namespace": "ai-tenant-e2e-mt-8d991807", "name": "odh-trusted-ca-bundle", "reconcileID": "72e9d64e-feae-449e-8ace-8e27242719b5", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-8d991807"} 2026-07-09T03:12:37Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-8d991807"}, "namespace": "ai-tenant-e2e-mt-8d991807", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "98b3d172-7282-4548-9923-807595c3383a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-8d991807"} 2026-07-09T03:12:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-e9c8e886"}, "namespace": "ai-tenant-e2e-mt-e9c8e886", "name": "openshift-service-ca.crt", "reconcileID": "0f58ed64-7e9e-449e-b4ef-a8b9fa3c24f7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-e9c8e886"} 2026-07-09T03:12:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-e9c8e886"}, "namespace": "ai-tenant-e2e-mt-e9c8e886", "name": "odh-trusted-ca-bundle", "reconcileID": "b0b092de-e126-4695-865b-a51bc495811a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-e9c8e886"} 2026-07-09T03:12:38Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-e9c8e886"}, "namespace": "ai-tenant-e2e-mt-e9c8e886", "name": "openshift-service-ca.crt", "reconcileID": "9fe0be69-77f3-4800-8e16-724ee0fbeb32", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-e9c8e886", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:12:38Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-e9c8e886"}, "namespace": "ai-tenant-e2e-mt-e9c8e886", "name": "openshift-service-ca.crt", "reconcileID": "9fe0be69-77f3-4800-8e16-724ee0fbeb32", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-e9c8e886"} 2026-07-09T03:12:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-e9c8e886"}, "namespace": "ai-tenant-e2e-mt-e9c8e886", "name": "openshift-service-ca.crt", "reconcileID": "0ed1eaa5-9008-4ab5-81c2-70d14e9b96f4", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-e9c8e886"} 2026-07-09T03:12:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-e9c8e886"}, "namespace": "ai-tenant-e2e-mt-e9c8e886", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b6dc520c-5ba8-4bae-91f9-3ca50b813392", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-e9c8e886"} 2026-07-09T03:12:38Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-e9c8e886"}, "namespace": "ai-tenant-e2e-mt-e9c8e886", "name": "odh-trusted-ca-bundle", "reconcileID": "0a25954b-29bb-4e87-a9f4-cbf993df4aeb", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-e9c8e886", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:12:38Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-e9c8e886"}, "namespace": "ai-tenant-e2e-mt-e9c8e886", "name": "odh-trusted-ca-bundle", "reconcileID": "0a25954b-29bb-4e87-a9f4-cbf993df4aeb", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-e9c8e886"} 2026-07-09T03:12:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-e9c8e886"}, "namespace": "ai-tenant-e2e-mt-e9c8e886", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "880bbd4e-2db4-4642-be1e-f9e68451075d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-e9c8e886"} 2026-07-09T03:12:45Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-8d991807"}, "namespace": "ai-tenant-e2e-mt-8d991807", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "90349e1f-e3fd-4619-b152-40d3867332e8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-8d991807"} 2026-07-09T03:12:45Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-8d991807"}, "namespace": "ai-tenant-e2e-mt-8d991807", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "90349e1f-e3fd-4619-b152-40d3867332e8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-8d991807", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:12:45Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-8d991807"}, "namespace": "ai-tenant-e2e-mt-8d991807", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "90349e1f-e3fd-4619-b152-40d3867332e8", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-8d991807 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T03:12:45Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-8d991807"}, "namespace": "ai-tenant-e2e-mt-8d991807", "name": "odh-trusted-ca-bundle", "reconcileID": "759c3529-5766-4b2a-810b-0aeeb6e3dba6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-8d991807"} 2026-07-09T03:12:45Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-8d991807"}, "namespace": "ai-tenant-e2e-mt-8d991807", "name": "odh-trusted-ca-bundle", "reconcileID": "759c3529-5766-4b2a-810b-0aeeb6e3dba6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-8d991807"} 2026-07-09T03:12:45Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-8d991807"}, "namespace": "ai-tenant-e2e-mt-8d991807", "name": "openshift-service-ca.crt", "reconcileID": "0af9d781-4e0e-4187-85dd-782a3cf40c11", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-8d991807"} 2026-07-09T03:12:45Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-8d991807"}, "namespace": "ai-tenant-e2e-mt-8d991807", "name": "openshift-service-ca.crt", "reconcileID": "0af9d781-4e0e-4187-85dd-782a3cf40c11", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-8d991807"} 2026-07-09T03:12:45Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-8d991807"}, "namespace": "ai-tenant-e2e-mt-8d991807", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5235a384-df24-42ae-bd8b-2d781ab8bdee", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-8d991807"} 2026-07-09T03:12:45Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-8d991807"}, "namespace": "ai-tenant-e2e-mt-8d991807", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5235a384-df24-42ae-bd8b-2d781ab8bdee", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-8d991807"} 2026-07-09T03:12:51Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-e9c8e886"}, "namespace": "ai-tenant-e2e-mt-e9c8e886", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e1b47636-90c4-4452-96b3-05e80515ff76", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-e9c8e886"} 2026-07-09T03:12:51Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-e9c8e886"}, "namespace": "ai-tenant-e2e-mt-e9c8e886", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e1b47636-90c4-4452-96b3-05e80515ff76", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-e9c8e886", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:12:51Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-e9c8e886"}, "namespace": "ai-tenant-e2e-mt-e9c8e886", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e1b47636-90c4-4452-96b3-05e80515ff76", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-e9c8e886 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T03:12:51Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-e9c8e886"}, "namespace": "ai-tenant-e2e-mt-e9c8e886", "name": "odh-trusted-ca-bundle", "reconcileID": "597bb2eb-785c-459d-92fa-5276258f044f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-e9c8e886"} 2026-07-09T03:12:51Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-e9c8e886"}, "namespace": "ai-tenant-e2e-mt-e9c8e886", "name": "odh-trusted-ca-bundle", "reconcileID": "597bb2eb-785c-459d-92fa-5276258f044f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-e9c8e886"} 2026-07-09T03:12:51Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-e9c8e886"}, "namespace": "ai-tenant-e2e-mt-e9c8e886", "name": "openshift-service-ca.crt", "reconcileID": "3bb1e012-6859-42e2-a202-923f046efa31", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-e9c8e886"} 2026-07-09T03:12:51Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-e9c8e886"}, "namespace": "ai-tenant-e2e-mt-e9c8e886", "name": "openshift-service-ca.crt", "reconcileID": "3bb1e012-6859-42e2-a202-923f046efa31", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-e9c8e886"} 2026-07-09T03:12:51Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-e9c8e886"}, "namespace": "ai-tenant-e2e-mt-e9c8e886", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3592e712-486e-4c93-82b2-b347203317bf", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-e9c8e886"} 2026-07-09T03:12:51Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-e9c8e886"}, "namespace": "ai-tenant-e2e-mt-e9c8e886", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3592e712-486e-4c93-82b2-b347203317bf", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-e9c8e886"} 2026-07-09T03:12:52Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-788622c4"}, "namespace": "ai-tenant-e2e-mt-788622c4", "name": "openshift-service-ca.crt", "reconcileID": "7153d7e8-b3b4-4bf6-9362-b52a7a081766", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-788622c4"} 2026-07-09T03:12:52Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-788622c4"}, "namespace": "ai-tenant-e2e-mt-788622c4", "name": "odh-trusted-ca-bundle", "reconcileID": "75759fe6-62ca-42d2-a1dc-51d9ed1ee9ce", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-788622c4"} 2026-07-09T03:12:52Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-788622c4"}, "namespace": "ai-tenant-e2e-mt-788622c4", "name": "openshift-service-ca.crt", "reconcileID": "0e1068c5-dd4d-4329-8588-708ba5c08fa8", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-788622c4", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:12:52Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-788622c4"}, "namespace": "ai-tenant-e2e-mt-788622c4", "name": "openshift-service-ca.crt", "reconcileID": "0e1068c5-dd4d-4329-8588-708ba5c08fa8", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-788622c4"} 2026-07-09T03:12:52Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-788622c4"}, "namespace": "ai-tenant-e2e-mt-788622c4", "name": "openshift-service-ca.crt", "reconcileID": "235ddad5-ef7a-4c7f-8804-a822dbe788ab", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-788622c4"} 2026-07-09T03:12:52Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-788622c4"}, "namespace": "ai-tenant-e2e-mt-788622c4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bb031bd1-9aff-42f5-ac48-709f9d76b977", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-788622c4"} 2026-07-09T03:12:52Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-788622c4"}, "namespace": "ai-tenant-e2e-mt-788622c4", "name": "odh-trusted-ca-bundle", "reconcileID": "ca8efdb9-d161-44fb-8788-1b1c90339840", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-788622c4", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:12:52Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-788622c4"}, "namespace": "ai-tenant-e2e-mt-788622c4", "name": "odh-trusted-ca-bundle", "reconcileID": "ca8efdb9-d161-44fb-8788-1b1c90339840", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-788622c4"} 2026-07-09T03:12:52Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-788622c4"}, "namespace": "ai-tenant-e2e-mt-788622c4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "505dd024-2824-4cc2-9196-25437536fba3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-788622c4"} 2026-07-09T03:13:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-788622c4"}, "namespace": "ai-tenant-e2e-mt-788622c4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0b4f18c7-5007-4fb7-a81a-74c05da46268", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-788622c4"} 2026-07-09T03:13:29Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-788622c4"}, "namespace": "ai-tenant-e2e-mt-788622c4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0b4f18c7-5007-4fb7-a81a-74c05da46268", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-788622c4", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:13:29Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-788622c4"}, "namespace": "ai-tenant-e2e-mt-788622c4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0b4f18c7-5007-4fb7-a81a-74c05da46268", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-788622c4 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T03:13:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-788622c4"}, "namespace": "ai-tenant-e2e-mt-788622c4", "name": "odh-trusted-ca-bundle", "reconcileID": "33f8779e-e76a-4bf9-bac9-d7270a7cefb8", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-788622c4"} 2026-07-09T03:13:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-788622c4"}, "namespace": "ai-tenant-e2e-mt-788622c4", "name": "odh-trusted-ca-bundle", "reconcileID": "33f8779e-e76a-4bf9-bac9-d7270a7cefb8", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-788622c4"} 2026-07-09T03:13:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-788622c4"}, "namespace": "ai-tenant-e2e-mt-788622c4", "name": "openshift-service-ca.crt", "reconcileID": "e15d4820-96bb-42ed-a00f-e98e98964569", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-788622c4"} 2026-07-09T03:13:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-788622c4"}, "namespace": "ai-tenant-e2e-mt-788622c4", "name": "openshift-service-ca.crt", "reconcileID": "e15d4820-96bb-42ed-a00f-e98e98964569", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-788622c4"} 2026-07-09T03:13:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-788622c4"}, "namespace": "ai-tenant-e2e-mt-788622c4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8c58ace2-65d0-43ac-8431-10e5cea19b22", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-788622c4"} 2026-07-09T03:13:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-788622c4"}, "namespace": "ai-tenant-e2e-mt-788622c4", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8c58ace2-65d0-43ac-8431-10e5cea19b22", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-788622c4"} 2026-07-09T03:13:30Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "5415085b-7ed6-4c18-860c-b77cd297bff5", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:13:30Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "5415085b-7ed6-4c18-860c-b77cd297bff5", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835-authn-ssl"} 2026-07-09T03:13:30Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "5415085b-7ed6-4c18-860c-b77cd297bff5", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:13:30Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "6a7cf832-61d6-49e5-9c4e-35e0f0562cdc", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:13:30Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "6a7cf832-61d6-49e5-9c4e-35e0f0562cdc", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835-authn-ssl"} 2026-07-09T03:13:30Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "6a7cf832-61d6-49e5-9c4e-35e0f0562cdc", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:13:30Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "63b2cb90-120b-45e6-952e-73b4c7c06289", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:13:30Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "63b2cb90-120b-45e6-952e-73b4c7c06289", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835-authn-ssl"} 2026-07-09T03:13:30Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "63b2cb90-120b-45e6-952e-73b4c7c06289", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:13:30Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "e1f844d4-508d-4e6e-8dfa-ab576d396a77", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:13:30Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "e1f844d4-508d-4e6e-8dfa-ab576d396a77", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835-authn-ssl"} 2026-07-09T03:13:30Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "e1f844d4-508d-4e6e-8dfa-ab576d396a77", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:13:30Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "cc885ba0-9e83-4656-9728-873b5f43f75e", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:13:30Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "cc885ba0-9e83-4656-9728-873b5f43f75e", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835-authn-ssl"} 2026-07-09T03:13:30Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "cc885ba0-9e83-4656-9728-873b5f43f75e", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:13:30Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "cf981bd5-ce2b-48e2-bd5b-ff7821fd8bb6", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:13:30Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "cf981bd5-ce2b-48e2-bd5b-ff7821fd8bb6", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835-authn-ssl"} 2026-07-09T03:13:30Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "cf981bd5-ce2b-48e2-bd5b-ff7821fd8bb6", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:13:31Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "28d1f593-d63b-4171-8f77-b62309ceb4ef", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:13:31Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "28d1f593-d63b-4171-8f77-b62309ceb4ef", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835-authn-ssl"} 2026-07-09T03:13:31Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "28d1f593-d63b-4171-8f77-b62309ceb4ef", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:13:33Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "openshift-service-ca.crt", "reconcileID": "1ed21e30-6a6d-4673-aabb-b7247add9d44", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:13:33Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "odh-trusted-ca-bundle", "reconcileID": "642122db-06aa-4069-a489-fd3b2f6d30e6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:13:33Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "openshift-service-ca.crt", "reconcileID": "baeb8140-d7f2-4205-96ca-c21f9ed2c3ff", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-cf3835", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:13:33Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "openshift-service-ca.crt", "reconcileID": "baeb8140-d7f2-4205-96ca-c21f9ed2c3ff", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:13:33Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0c03aafd-8ab3-40a5-a062-55fa12b7936d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:13:33Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "openshift-service-ca.crt", "reconcileID": "d78da622-e5cf-4d82-86f0-30b2dfab7171", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:13:33Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "odh-trusted-ca-bundle", "reconcileID": "f793048c-cb62-47db-adc3-d82521c764ec", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-cf3835", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:13:33Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "odh-trusted-ca-bundle", "reconcileID": "f793048c-cb62-47db-adc3-d82521c764ec", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:13:33Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "15ad8450-27ad-4156-8886-a50eb68d5d9e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:13:34Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "d3d3c238-fa15-422d-b433-74ff420ce4d1", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:13:34Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "d3d3c238-fa15-422d-b433-74ff420ce4d1", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835-authn-ssl"} 2026-07-09T03:13:34Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "d3d3c238-fa15-422d-b433-74ff420ce4d1", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:13:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "71cafafe-f8af-477d-a26c-94d24841d014", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-09T03:13:38Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "56097051-0ed7-4bbf-8883-b8322a15d0aa", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:13:38Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "56097051-0ed7-4bbf-8883-b8322a15d0aa", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37-authn-ssl"} 2026-07-09T03:13:38Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "56097051-0ed7-4bbf-8883-b8322a15d0aa", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:13:38Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "de4706aa-3de8-473e-89bb-e0c2c141445a", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:13:38Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "de4706aa-3de8-473e-89bb-e0c2c141445a", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37-authn-ssl"} 2026-07-09T03:13:38Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "de4706aa-3de8-473e-89bb-e0c2c141445a", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:13:38Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "b703438d-4561-48bf-b7b7-ab43cf2e4848", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:13:38Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "b703438d-4561-48bf-b7b7-ab43cf2e4848", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37-authn-ssl"} 2026-07-09T03:13:38Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "b703438d-4561-48bf-b7b7-ab43cf2e4848", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:13:38Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "c9935d87-3589-4982-9502-046b9e2e5db6", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:13:38Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "c9935d87-3589-4982-9502-046b9e2e5db6", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37-authn-ssl"} 2026-07-09T03:13:38Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "c9935d87-3589-4982-9502-046b9e2e5db6", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:13:38Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "015e926c-a9ec-4c31-b53a-9e4d04193633", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:13:38Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "015e926c-a9ec-4c31-b53a-9e4d04193633", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37-authn-ssl"} 2026-07-09T03:13:38Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "015e926c-a9ec-4c31-b53a-9e4d04193633", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:13:38Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "64c30f51-e5e6-4f34-baa7-0758405eb9b1", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:13:38Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "64c30f51-e5e6-4f34-baa7-0758405eb9b1", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37-authn-ssl"} 2026-07-09T03:13:38Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "64c30f51-e5e6-4f34-baa7-0758405eb9b1", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:13:42Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "01cf97e4-2504-4d30-aaad-1d0b22f798b9", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:13:42Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "01cf97e4-2504-4d30-aaad-1d0b22f798b9", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835-authn-ssl"} 2026-07-09T03:13:42Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "01cf97e4-2504-4d30-aaad-1d0b22f798b9", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:13:42Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "87b1daa6-a97a-4f74-9cfc-f2297d5f5927", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:13:42Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "87b1daa6-a97a-4f74-9cfc-f2297d5f5927", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37-authn-ssl"} 2026-07-09T03:13:42Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "87b1daa6-a97a-4f74-9cfc-f2297d5f5927", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:13:44Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "openshift-service-ca.crt", "reconcileID": "0e0a1758-f4de-4e2c-8619-d85d0bd61003", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:13:44Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "openshift-service-ca.crt", "reconcileID": "6290b0c3-024a-4f06-befd-2984dc6c1dd7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-defa37", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:13:44Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "openshift-service-ca.crt", "reconcileID": "6290b0c3-024a-4f06-befd-2984dc6c1dd7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:13:44Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "odh-trusted-ca-bundle", "reconcileID": "68a25439-e958-4e77-b80f-c403bfe61eab", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-defa37", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:13:44Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "odh-trusted-ca-bundle", "reconcileID": "68a25439-e958-4e77-b80f-c403bfe61eab", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T03:13:44Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "openshift-service-ca.crt", "reconcileID": "fc40a399-d97f-4093-8bd3-0f95d30d98b3", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:13:44Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "baab46a5-9fd2-4a03-8831-5ff1d161600a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:13:44Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "odh-trusted-ca-bundle", "reconcileID": "9fceaf38-8b6f-46ed-9580-d7eb33a2f2e8", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:13:44Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "odh-trusted-ca-bundle", "reconcileID": "46bfad46-59c8-4441-be0a-9a8cb78c22cd", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-defa37", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:13:44Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "odh-trusted-ca-bundle", "reconcileID": "46bfad46-59c8-4441-be0a-9a8cb78c22cd", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:13:44Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "67fca5bc-32a5-49fe-bfb1-b48ada584647", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:13:44Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "3c20ae7c-8281-46c0-9bf7-5b6e216a760f", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:13:44Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "3c20ae7c-8281-46c0-9bf7-5b6e216a760f", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37-authn-ssl"} 2026-07-09T03:13:44Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "3c20ae7c-8281-46c0-9bf7-5b6e216a760f", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:13:45Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "4ee22375-03b7-429e-96fa-b247b9155a9c", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-09T03:13:48Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "888a1dfd-fdf0-4ac1-a136-eec0a29e6060", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:13:48Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "888a1dfd-fdf0-4ac1-a136-eec0a29e6060", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37-authn-ssl"} 2026-07-09T03:13:48Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "888a1dfd-fdf0-4ac1-a136-eec0a29e6060", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:13:49Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "150a5a16-1ef9-454e-b38c-6df0b1e0532f", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:13:49Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "150a5a16-1ef9-454e-b38c-6df0b1e0532f", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:13:49Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "150a5a16-1ef9-454e-b38c-6df0b1e0532f", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:13:49Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "0501ae9a-6d14-4852-92e5-19bc661e0439", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:13:49Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "150a5a16-1ef9-454e-b38c-6df0b1e0532f", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835-kserve-route-authn"} 2026-07-09T03:13:49Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "150a5a16-1ef9-454e-b38c-6df0b1e0532f", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835", "authType": "userdefined"} 2026-07-09T03:13:49Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "150a5a16-1ef9-454e-b38c-6df0b1e0532f", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:13:49Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "0501ae9a-6d14-4852-92e5-19bc661e0439", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835-authn-ssl"} 2026-07-09T03:13:49Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "0501ae9a-6d14-4852-92e5-19bc661e0439", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:13:49Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "d2b09a35-391e-4e51-9291-bed0239d748b", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:13:49Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "d2b09a35-391e-4e51-9291-bed0239d748b", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:13:49Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "d2b09a35-391e-4e51-9291-bed0239d748b", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:13:49Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "d2b09a35-391e-4e51-9291-bed0239d748b", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835-kserve-route-authn"} 2026-07-09T03:13:49Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "d2b09a35-391e-4e51-9291-bed0239d748b", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835", "authType": "userdefined"} 2026-07-09T03:13:49Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "d2b09a35-391e-4e51-9291-bed0239d748b", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:13:55Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "7904f438-6cb5-49da-9469-28f237609d14", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:13:55Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "7904f438-6cb5-49da-9469-28f237609d14", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835-authn-ssl"} 2026-07-09T03:13:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "7904f438-6cb5-49da-9469-28f237609d14", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:13:57Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "75cd4f94-6fa6-4f83-a503-586f424c8c56", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:13:57Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "75cd4f94-6fa6-4f83-a503-586f424c8c56", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:13:57Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "75cd4f94-6fa6-4f83-a503-586f424c8c56", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:13:57Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "75cd4f94-6fa6-4f83-a503-586f424c8c56", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835-kserve-route-authn"} 2026-07-09T03:13:57Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "75cd4f94-6fa6-4f83-a503-586f424c8c56", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835", "authType": "userdefined"} 2026-07-09T03:13:57Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "75cd4f94-6fa6-4f83-a503-586f424c8c56", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:14:00Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "27e22638-e5f4-4f12-9106-7d479f4fcd54", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:14:00Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "27e22638-e5f4-4f12-9106-7d479f4fcd54", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:14:00Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "27e22638-e5f4-4f12-9106-7d479f4fcd54", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:14:00Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "27e22638-e5f4-4f12-9106-7d479f4fcd54", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835-kserve-route-authn"} 2026-07-09T03:14:00Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "27e22638-e5f4-4f12-9106-7d479f4fcd54", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835", "authType": "userdefined"} 2026-07-09T03:14:00Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "27e22638-e5f4-4f12-9106-7d479f4fcd54", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:14:05Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "c47ca49c-b2dd-43d8-ba60-edba662a3e64", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:14:05Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "c47ca49c-b2dd-43d8-ba60-edba662a3e64", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835-authn-ssl"} 2026-07-09T03:14:05Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "c47ca49c-b2dd-43d8-ba60-edba662a3e64", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:14:19Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "ca7a702e-689d-4137-9674-f6b194f2d969", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:14:19Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "ca7a702e-689d-4137-9674-f6b194f2d969", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:14:19Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "ca7a702e-689d-4137-9674-f6b194f2d969", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:14:19Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "ca7a702e-689d-4137-9674-f6b194f2d969", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835-kserve-route-authn"} 2026-07-09T03:14:19Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "ca7a702e-689d-4137-9674-f6b194f2d969", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835", "authType": "userdefined"} 2026-07-09T03:14:19Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "ca7a702e-689d-4137-9674-f6b194f2d969", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:14:21Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "59d5c530-368b-4c6e-8f76-94cbbad79a20", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:14:21Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "59d5c530-368b-4c6e-8f76-94cbbad79a20", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:14:21Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "59d5c530-368b-4c6e-8f76-94cbbad79a20", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:14:21Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "f3432425-0424-4875-9284-c182b667b710", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:14:21Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "59d5c530-368b-4c6e-8f76-94cbbad79a20", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37-kserve-route-authn"} 2026-07-09T03:14:21Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "59d5c530-368b-4c6e-8f76-94cbbad79a20", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37", "authType": "userdefined"} 2026-07-09T03:14:21Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "59d5c530-368b-4c6e-8f76-94cbbad79a20", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:14:21Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "f3432425-0424-4875-9284-c182b667b710", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37-authn-ssl"} 2026-07-09T03:14:21Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "f3432425-0424-4875-9284-c182b667b710", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:14:21Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "e7aef3d1-cbe7-4320-b99a-b43638e84546", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:14:21Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "e7aef3d1-cbe7-4320-b99a-b43638e84546", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:14:21Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "e7aef3d1-cbe7-4320-b99a-b43638e84546", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:14:21Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "e7aef3d1-cbe7-4320-b99a-b43638e84546", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37-kserve-route-authn"} 2026-07-09T03:14:21Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "e7aef3d1-cbe7-4320-b99a-b43638e84546", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37", "authType": "userdefined"} 2026-07-09T03:14:21Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "e7aef3d1-cbe7-4320-b99a-b43638e84546", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:14:35Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "5a4e729c-e829-48d1-981b-7eb84422a296", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:14:35Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "5a4e729c-e829-48d1-981b-7eb84422a296", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37-authn-ssl"} 2026-07-09T03:14:35Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "5a4e729c-e829-48d1-981b-7eb84422a296", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:14:37Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "65364e1d-050d-4f47-baaf-f9128973b750", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:14:37Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "65364e1d-050d-4f47-baaf-f9128973b750", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:14:37Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "65364e1d-050d-4f47-baaf-f9128973b750", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:14:37Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "65364e1d-050d-4f47-baaf-f9128973b750", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37-kserve-route-authn"} 2026-07-09T03:14:37Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "65364e1d-050d-4f47-baaf-f9128973b750", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37", "authType": "userdefined"} 2026-07-09T03:14:37Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "65364e1d-050d-4f47-baaf-f9128973b750", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:14:41Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "92e8c1d4-3834-4bda-ae68-e7ad084b3f9b", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:14:41Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "92e8c1d4-3834-4bda-ae68-e7ad084b3f9b", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:14:41Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "92e8c1d4-3834-4bda-ae68-e7ad084b3f9b", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:14:41Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "92e8c1d4-3834-4bda-ae68-e7ad084b3f9b", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37-kserve-route-authn"} 2026-07-09T03:14:41Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "92e8c1d4-3834-4bda-ae68-e7ad084b3f9b", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37", "authType": "userdefined"} 2026-07-09T03:14:41Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "92e8c1d4-3834-4bda-ae68-e7ad084b3f9b", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:14:42Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "e74b195a-48fc-4995-9efe-63bf2e811c2d", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:14:42Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "e74b195a-48fc-4995-9efe-63bf2e811c2d", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37-authn-ssl"} 2026-07-09T03:14:42Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "e74b195a-48fc-4995-9efe-63bf2e811c2d", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:14:53Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "e4d4a93d-fb83-4c85-9e08-08df4e1754df", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:14:53Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "e4d4a93d-fb83-4c85-9e08-08df4e1754df", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:14:53Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "e4d4a93d-fb83-4c85-9e08-08df4e1754df", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:14:53Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "e4d4a93d-fb83-4c85-9e08-08df4e1754df", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37-kserve-route-authn"} 2026-07-09T03:14:53Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "e4d4a93d-fb83-4c85-9e08-08df4e1754df", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37", "authType": "userdefined"} 2026-07-09T03:14:53Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "e4d4a93d-fb83-4c85-9e08-08df4e1754df", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:15:03Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "dabc27f1-de49-4f7a-9356-02fe8173a038", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:15:03Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "dabc27f1-de49-4f7a-9356-02fe8173a038", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835"} 2026-07-09T03:15:03Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "dabc27f1-de49-4f7a-9356-02fe8173a038", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:15:03Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "dabc27f1-de49-4f7a-9356-02fe8173a038", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:15:03Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "dabc27f1-de49-4f7a-9356-02fe8173a038", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835-kserve-route-authn"} 2026-07-09T03:15:03Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "dabc27f1-de49-4f7a-9356-02fe8173a038", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835-kserve-route-authn"} 2026-07-09T03:15:03Z DEBUG Triggering LLMInferenceService Cleanup for Namespace {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "dabc27f1-de49-4f7a-9356-02fe8173a038", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:15:03Z DEBUG Cleaning up LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "dabc27f1-de49-4f7a-9356-02fe8173a038", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:15:03Z INFO Stop LLMInferenceService reconciliation {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-cf3835","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "test-model-cf3835", "reconcileID": "31d99019-a4e1-49ba-b796-95d015fe1f7b", "LLMInferenceService": "test-model-cf3835", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:15:03Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "abeb7c83-6b9b-43ba-b71f-38d4a62c9c5f", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:15:03Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "abeb7c83-6b9b-43ba-b71f-38d4a62c9c5f", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835-authn-ssl"} 2026-07-09T03:15:03Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "abeb7c83-6b9b-43ba-b71f-38d4a62c9c5f", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:15:03Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "300d7c97-7ced-45bf-88bd-c97fea85f2d6", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:15:03Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "300d7c97-7ced-45bf-88bd-c97fea85f2d6", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835-authn-ssl"} 2026-07-09T03:15:03Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "300d7c97-7ced-45bf-88bd-c97fea85f2d6", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:15:03Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "f216a353-877b-44dc-a087-e289f4a3f744", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:15:03Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "f216a353-877b-44dc-a087-e289f4a3f744", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37"} 2026-07-09T03:15:03Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "f216a353-877b-44dc-a087-e289f4a3f744", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:15:03Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "f216a353-877b-44dc-a087-e289f4a3f744", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:15:03Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "f216a353-877b-44dc-a087-e289f4a3f744", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37-kserve-route-authn"} 2026-07-09T03:15:03Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "f216a353-877b-44dc-a087-e289f4a3f744", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37-kserve-route-authn"} 2026-07-09T03:15:03Z DEBUG Triggering LLMInferenceService Cleanup for Namespace {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "f216a353-877b-44dc-a087-e289f4a3f744", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:15:03Z DEBUG Cleaning up LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "f216a353-877b-44dc-a087-e289f4a3f744", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:15:04Z INFO Stop LLMInferenceService reconciliation {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-defa37","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "test-model-defa37", "reconcileID": "dfabfad2-550d-4163-beba-b14efc634b4f", "LLMInferenceService": "test-model-defa37", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:15:04Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "5305ed28-b94e-4cd5-a0b1-4bf59aad9ab6", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:15:04Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "5305ed28-b94e-4cd5-a0b1-4bf59aad9ab6", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37-authn-ssl"} 2026-07-09T03:15:04Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "5305ed28-b94e-4cd5-a0b1-4bf59aad9ab6", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:15:04Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "003f17a7-854a-43de-8279-d008a95116f6", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:15:04Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "003f17a7-854a-43de-8279-d008a95116f6", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37-authn-ssl"} 2026-07-09T03:15:04Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "003f17a7-854a-43de-8279-d008a95116f6", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:15:04Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "87aab07a-99b5-4c17-ad81-be85382fb5a6", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:15:04Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "87aab07a-99b5-4c17-ad81-be85382fb5a6", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835-authn-ssl"} 2026-07-09T03:15:04Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-cf3835","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-cf3835", "reconcileID": "87aab07a-99b5-4c17-ad81-be85382fb5a6", "Gateway": "e2e-inf-a-cf3835", "namespace": "openshift-ingress"} 2026-07-09T03:15:42Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f6d24a9a-57ee-4531-864e-7d56587f2b30", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:15:42Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f6d24a9a-57ee-4531-864e-7d56587f2b30", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-cf3835", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:15:42Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f6d24a9a-57ee-4531-864e-7d56587f2b30", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-inf-a-cf3835 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T03:15:42Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "odh-trusted-ca-bundle", "reconcileID": "d05f907b-4f37-410b-887e-2c18c816ed07", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:15:42Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "odh-trusted-ca-bundle", "reconcileID": "d05f907b-4f37-410b-887e-2c18c816ed07", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:15:42Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "openshift-service-ca.crt", "reconcileID": "eb3668a6-0fc4-490c-9a34-04bc874a6a13", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:15:42Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "openshift-service-ca.crt", "reconcileID": "eb3668a6-0fc4-490c-9a34-04bc874a6a13", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:15:42Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "96b8e357-4824-4018-8ef4-c4f6285b05cb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:15:42Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-cf3835"}, "namespace": "ai-tenant-e2e-inf-a-cf3835", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "96b8e357-4824-4018-8ef4-c4f6285b05cb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-cf3835"} 2026-07-09T03:15:43Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "7bc4233d-6996-46d6-87d6-ff92e11c19a7", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:15:43Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "7bc4233d-6996-46d6-87d6-ff92e11c19a7", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37-authn-ssl"} 2026-07-09T03:15:43Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-defa37","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-defa37", "reconcileID": "7bc4233d-6996-46d6-87d6-ff92e11c19a7", "Gateway": "e2e-inf-b-defa37", "namespace": "openshift-ingress"} 2026-07-09T03:15:48Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7a04209f-3df7-4c98-89a7-afa05691b90d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:15:48Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7a04209f-3df7-4c98-89a7-afa05691b90d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-defa37", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:15:48Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7a04209f-3df7-4c98-89a7-afa05691b90d", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-inf-b-defa37 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T03:15:48Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "odh-trusted-ca-bundle", "reconcileID": "4ea4fa71-3905-4b0a-b695-622b324a535f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:15:48Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "odh-trusted-ca-bundle", "reconcileID": "4ea4fa71-3905-4b0a-b695-622b324a535f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-defa37", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:15:48Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "odh-trusted-ca-bundle", "reconcileID": "4ea4fa71-3905-4b0a-b695-622b324a535f", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-inf-b-defa37 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T03:15:48Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "openshift-service-ca.crt", "reconcileID": "4c0c9d9d-acbe-4efc-bb2a-379623a4b733", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:15:48Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "openshift-service-ca.crt", "reconcileID": "4c0c9d9d-acbe-4efc-bb2a-379623a4b733", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:15:48Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d5b978bc-a006-4416-8303-411e6df643e1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:15:48Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d5b978bc-a006-4416-8303-411e6df643e1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:15:48Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "odh-trusted-ca-bundle", "reconcileID": "94d92947-e5b7-4075-89b7-498608ae5e5e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:15:48Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-defa37"}, "namespace": "ai-tenant-e2e-inf-b-defa37", "name": "odh-trusted-ca-bundle", "reconcileID": "94d92947-e5b7-4075-89b7-498608ae5e5e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-defa37"} 2026-07-09T03:15:50Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-8d5c89","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89", "reconcileID": "ba652bda-bcb2-4d7c-8342-e53aa55ab89c", "Gateway": "e2e-api-a-8d5c89", "namespace": "openshift-ingress"} 2026-07-09T03:15:50Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-8d5c89","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89", "reconcileID": "ba652bda-bcb2-4d7c-8342-e53aa55ab89c", "Gateway": "e2e-api-a-8d5c89", "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89-authn-ssl"} 2026-07-09T03:15:50Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-8d5c89","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89", "reconcileID": "ba652bda-bcb2-4d7c-8342-e53aa55ab89c", "Gateway": "e2e-api-a-8d5c89", "namespace": "openshift-ingress"} 2026-07-09T03:15:50Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-8d5c89","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89", "reconcileID": "a3db6173-d0c6-4437-a5e3-e46fdb475aa1", "Gateway": "e2e-api-a-8d5c89", "namespace": "openshift-ingress"} 2026-07-09T03:15:50Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-8d5c89","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89", "reconcileID": "a3db6173-d0c6-4437-a5e3-e46fdb475aa1", "Gateway": "e2e-api-a-8d5c89", "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89-authn-ssl"} 2026-07-09T03:15:50Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-8d5c89","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89", "reconcileID": "a3db6173-d0c6-4437-a5e3-e46fdb475aa1", "Gateway": "e2e-api-a-8d5c89", "namespace": "openshift-ingress"} 2026-07-09T03:15:50Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-8d5c89","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89", "reconcileID": "39d5adb1-8101-4846-bf49-3f5ff2d66e31", "Gateway": "e2e-api-a-8d5c89", "namespace": "openshift-ingress"} 2026-07-09T03:15:50Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-8d5c89","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89", "reconcileID": "39d5adb1-8101-4846-bf49-3f5ff2d66e31", "Gateway": "e2e-api-a-8d5c89", "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89-authn-ssl"} 2026-07-09T03:15:50Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-8d5c89","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89", "reconcileID": "39d5adb1-8101-4846-bf49-3f5ff2d66e31", "Gateway": "e2e-api-a-8d5c89", "namespace": "openshift-ingress"} 2026-07-09T03:15:50Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-8d5c89","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89", "reconcileID": "b47da48e-dcfc-4c8a-b41f-6f3bb64cdae1", "Gateway": "e2e-api-a-8d5c89", "namespace": "openshift-ingress"} 2026-07-09T03:15:50Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-8d5c89","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89", "reconcileID": "b47da48e-dcfc-4c8a-b41f-6f3bb64cdae1", "Gateway": "e2e-api-a-8d5c89", "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89-authn-ssl"} 2026-07-09T03:15:50Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-8d5c89","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89", "reconcileID": "b47da48e-dcfc-4c8a-b41f-6f3bb64cdae1", "Gateway": "e2e-api-a-8d5c89", "namespace": "openshift-ingress"} 2026-07-09T03:15:50Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-8d5c89","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89", "reconcileID": "3594980c-8e46-4aaf-a784-bb95b0230637", "Gateway": "e2e-api-a-8d5c89", "namespace": "openshift-ingress"} 2026-07-09T03:15:50Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-8d5c89","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89", "reconcileID": "3594980c-8e46-4aaf-a784-bb95b0230637", "Gateway": "e2e-api-a-8d5c89", "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89-authn-ssl"} 2026-07-09T03:15:50Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-8d5c89","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89", "reconcileID": "3594980c-8e46-4aaf-a784-bb95b0230637", "Gateway": "e2e-api-a-8d5c89", "namespace": "openshift-ingress"} 2026-07-09T03:15:50Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-8d5c89","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89", "reconcileID": "4bb1333e-9fc8-486b-9a92-81fccfd6bf92", "Gateway": "e2e-api-a-8d5c89", "namespace": "openshift-ingress"} 2026-07-09T03:15:50Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-8d5c89","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89", "reconcileID": "4bb1333e-9fc8-486b-9a92-81fccfd6bf92", "Gateway": "e2e-api-a-8d5c89", "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89-authn-ssl"} 2026-07-09T03:15:50Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-8d5c89","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89", "reconcileID": "4bb1333e-9fc8-486b-9a92-81fccfd6bf92", "Gateway": "e2e-api-a-8d5c89", "namespace": "openshift-ingress"} 2026-07-09T03:15:51Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-8d5c89","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89", "reconcileID": "c9c26f43-a722-4c6b-b065-2d19317b288d", "Gateway": "e2e-api-a-8d5c89", "namespace": "openshift-ingress"} 2026-07-09T03:15:51Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-8d5c89","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89", "reconcileID": "c9c26f43-a722-4c6b-b065-2d19317b288d", "Gateway": "e2e-api-a-8d5c89", "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89-authn-ssl"} 2026-07-09T03:15:51Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-8d5c89","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-8d5c89", "reconcileID": "c9c26f43-a722-4c6b-b065-2d19317b288d", "Gateway": "e2e-api-a-8d5c89", "namespace": "openshift-ingress"} 2026-07-09T03:15:52Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-8d5c89"}, "namespace": "ai-tenant-e2e-api-a-8d5c89", "name": "openshift-service-ca.crt", "reconcileID": "a854159e-4bd5-40b7-ae61-d749580e88af", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-8d5c89"} 2026-07-09T03:15:52Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-8d5c89"}, "namespace": "ai-tenant-e2e-api-a-8d5c89", "name": "odh-trusted-ca-bundle", "reconcileID": "7f304d26-63a8-42ea-8b6b-0164b29453df", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-8d5c89"} 2026-07-09T03:15:53Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-8d5c89"}, "namespace": "ai-tenant-e2e-api-a-8d5c89", "name": "openshift-service-ca.crt", "reconcileID": "b4190a4b-6217-4d18-9152-718f89fa88bc", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-8d5c89", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:15:53Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-8d5c89"}, "namespace": "ai-tenant-e2e-api-a-8d5c89", "name": "openshift-service-ca.crt", "reconcileID": "b4190a4b-6217-4d18-9152-718f89fa88bc", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-8d5c89"} 2026-07-09T03:15:53Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-8d5c89"}, "namespace": "ai-tenant-e2e-api-a-8d5c89", "name": "openshift-service-ca.crt", "reconcileID": "165ea799-c5b2-40b5-97ee-e0f4e911d650", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-8d5c89"} 2026-07-09T03:15:53Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-8d5c89"}, "namespace": "ai-tenant-e2e-api-a-8d5c89", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b63c5b0e-acfc-4301-97e1-eef583cb659d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-8d5c89"} 2026-07-09T03:15:53Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-8d5c89"}, "namespace": "ai-tenant-e2e-api-a-8d5c89", "name": "odh-trusted-ca-bundle", "reconcileID": "bb49bb24-7b99-4f2c-849e-00a096182146", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-8d5c89", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:15:53Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-8d5c89"}, "namespace": "ai-tenant-e2e-api-a-8d5c89", "name": "odh-trusted-ca-bundle", "reconcileID": "bb49bb24-7b99-4f2c-849e-00a096182146", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-8d5c89"} 2026-07-09T03:15:53Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-8d5c89"}, "namespace": "ai-tenant-e2e-api-a-8d5c89", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "949a3ca9-c7dd-438f-bc84-ebd5cea450c1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-8d5c89"} 2026-07-09T03:16:34Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-09T03:16:34Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-09T03:16:35Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-09T03:16:35Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-09T03:19:08Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-8d5c89"}, "namespace": "ai-tenant-e2e-api-a-8d5c89", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c12b0437-e832-4a9b-8d81-ed6c1689c4ed", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-8d5c89"} 2026-07-09T03:19:08Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-8d5c89"}, "namespace": "ai-tenant-e2e-api-a-8d5c89", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c12b0437-e832-4a9b-8d81-ed6c1689c4ed", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-8d5c89", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T03:19:08Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-8d5c89"}, "namespace": "ai-tenant-e2e-api-a-8d5c89", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c12b0437-e832-4a9b-8d81-ed6c1689c4ed", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-api-a-8d5c89 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T03:19:08Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-8d5c89"}, "namespace": "ai-tenant-e2e-api-a-8d5c89", "name": "odh-trusted-ca-bundle", "reconcileID": "23c84faf-e3d0-4cb9-8f4e-d7451b7bb4b3", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-8d5c89"} 2026-07-09T03:19:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-8d5c89"}, "namespace": "ai-tenant-e2e-api-a-8d5c89", "name": "odh-trusted-ca-bundle", "reconcileID": "23c84faf-e3d0-4cb9-8f4e-d7451b7bb4b3", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-8d5c89"} 2026-07-09T03:19:08Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-8d5c89"}, "namespace": "ai-tenant-e2e-api-a-8d5c89", "name": "openshift-service-ca.crt", "reconcileID": "98aeff61-9a90-4da8-8bbc-19b9962da33b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-8d5c89"} 2026-07-09T03:19:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-8d5c89"}, "namespace": "ai-tenant-e2e-api-a-8d5c89", "name": "openshift-service-ca.crt", "reconcileID": "98aeff61-9a90-4da8-8bbc-19b9962da33b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-8d5c89"} 2026-07-09T03:19:08Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-8d5c89"}, "namespace": "ai-tenant-e2e-api-a-8d5c89", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dbc491e2-d799-41d3-874f-8843238ca7b0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-8d5c89"} 2026-07-09T03:19:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-8d5c89"}, "namespace": "ai-tenant-e2e-api-a-8d5c89", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dbc491e2-d799-41d3-874f-8843238ca7b0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-8d5c89"}