2026-07-08T19:15:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-d5cdb3"}, "namespace": "e2e-webhook-test-d5cdb3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4ce220f6-c505-4623-a931-0704af187d41", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-d5cdb3"} 2026-07-08T19:15:06Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-d5cdb3"}, "namespace": "e2e-webhook-test-d5cdb3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "78d552a6-6de0-49c7-92df-daba04b00d2d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-d5cdb3"} 2026-07-08T19:15:06Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-d5cdb3"}, "namespace": "e2e-webhook-test-d5cdb3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "78d552a6-6de0-49c7-92df-daba04b00d2d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-d5cdb3", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:15:06Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-d5cdb3"}, "namespace": "e2e-webhook-test-d5cdb3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "78d552a6-6de0-49c7-92df-daba04b00d2d", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-test-d5cdb3 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T19:15:06Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-d5cdb3"}, "namespace": "e2e-webhook-test-d5cdb3", "name": "odh-trusted-ca-bundle", "reconcileID": "bcdb3dd7-d77a-4a8f-9fa4-575a2a97450a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-d5cdb3"} 2026-07-08T19:15:06Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-d5cdb3"}, "namespace": "e2e-webhook-test-d5cdb3", "name": "odh-trusted-ca-bundle", "reconcileID": "bcdb3dd7-d77a-4a8f-9fa4-575a2a97450a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-d5cdb3"} 2026-07-08T19:15:06Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-d5cdb3"}, "namespace": "e2e-webhook-test-d5cdb3", "name": "openshift-service-ca.crt", "reconcileID": "4d31bc57-fbf6-40f6-80f6-60d584a081c3", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-d5cdb3"} 2026-07-08T19:15:06Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-d5cdb3"}, "namespace": "e2e-webhook-test-d5cdb3", "name": "openshift-service-ca.crt", "reconcileID": "4d31bc57-fbf6-40f6-80f6-60d584a081c3", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-d5cdb3"} 2026-07-08T19:15:06Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-d5cdb3"}, "namespace": "e2e-webhook-test-d5cdb3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a30d4bd2-0eda-4fdb-9f9f-0bb1ece3eadf", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-d5cdb3"} 2026-07-08T19:15:06Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-d5cdb3"}, "namespace": "e2e-webhook-test-d5cdb3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a30d4bd2-0eda-4fdb-9f9f-0bb1ece3eadf", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-d5cdb3"} 2026-07-08T19:15:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-bed950"}, "namespace": "e2e-webhook-test-bed950", "name": "openshift-service-ca.crt", "reconcileID": "81ce2656-de79-427b-92be-ed2746641d22", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-bed950"} 2026-07-08T19:15:07Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-bed950"}, "namespace": "e2e-webhook-test-bed950", "name": "openshift-service-ca.crt", "reconcileID": "e585aee8-f362-43f1-a1a9-b045e4394a55", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-bed950", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:15:07Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-bed950"}, "namespace": "e2e-webhook-test-bed950", "name": "openshift-service-ca.crt", "reconcileID": "e585aee8-f362-43f1-a1a9-b045e4394a55", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-bed950"} 2026-07-08T19:15:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-bed950"}, "namespace": "e2e-webhook-test-bed950", "name": "odh-trusted-ca-bundle", "reconcileID": "ddab5fb2-b43d-4417-8951-1d32f4a5bb8b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-bed950"} 2026-07-08T19:15:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-bed950"}, "namespace": "e2e-webhook-test-bed950", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ef492a33-a682-4073-aec8-3bf1ef53cdde", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-bed950"} 2026-07-08T19:15:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-bed950"}, "namespace": "e2e-webhook-test-bed950", "name": "openshift-service-ca.crt", "reconcileID": "4172ab01-2428-4390-9bf8-7db837366527", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-bed950"} 2026-07-08T19:15:07Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-bed950"}, "namespace": "e2e-webhook-test-bed950", "name": "odh-trusted-ca-bundle", "reconcileID": "80738790-c8bf-4279-917c-dd26c5778a9b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-bed950", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:15:07Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-bed950"}, "namespace": "e2e-webhook-test-bed950", "name": "odh-trusted-ca-bundle", "reconcileID": "80738790-c8bf-4279-917c-dd26c5778a9b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-bed950"} 2026-07-08T19:15:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-bed950"}, "namespace": "e2e-webhook-test-bed950", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9fe929f1-e2aa-4751-95b9-48bf2bca06a7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-bed950"} 2026-07-08T19:15:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-bed950"}, "namespace": "e2e-webhook-test-bed950", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f47fc42a-5b4b-4e6c-bce0-c8481901e262", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-bed950"} 2026-07-08T19:15:13Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-bed950"}, "namespace": "e2e-webhook-test-bed950", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f47fc42a-5b4b-4e6c-bce0-c8481901e262", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-bed950", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:15:13Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-bed950"}, "namespace": "e2e-webhook-test-bed950", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f47fc42a-5b4b-4e6c-bce0-c8481901e262", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-test-bed950 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T19:15:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-bed950"}, "namespace": "e2e-webhook-test-bed950", "name": "odh-trusted-ca-bundle", "reconcileID": "1860c94d-6cbe-4359-9bd5-9ead899efe04", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-bed950"} 2026-07-08T19:15:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-bed950"}, "namespace": "e2e-webhook-test-bed950", "name": "odh-trusted-ca-bundle", "reconcileID": "1860c94d-6cbe-4359-9bd5-9ead899efe04", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-bed950"} 2026-07-08T19:15:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-bed950"}, "namespace": "e2e-webhook-test-bed950", "name": "openshift-service-ca.crt", "reconcileID": "3128569f-081f-4fb8-9951-0a44235d3048", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-bed950"} 2026-07-08T19:15:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-bed950"}, "namespace": "e2e-webhook-test-bed950", "name": "openshift-service-ca.crt", "reconcileID": "3128569f-081f-4fb8-9951-0a44235d3048", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-bed950"} 2026-07-08T19:15:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-bed950"}, "namespace": "e2e-webhook-test-bed950", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fb284f86-6a8d-46da-96a8-58c81573793a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-bed950"} 2026-07-08T19:15:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-bed950"}, "namespace": "e2e-webhook-test-bed950", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fb284f86-6a8d-46da-96a8-58c81573793a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-bed950"} 2026-07-08T19:24:32Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-08T19:24:32Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-08T19:24:32Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-08T19:24:32Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-08T19:26:02Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "a870fc07-4f91-4119-b84e-4c6ef12820bc", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-07-08T19:26:02Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "a870fc07-4f91-4119-b84e-4c6ef12820bc", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-07-08T19:26:02Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "a870fc07-4f91-4119-b84e-4c6ef12820bc", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-07-08T19:26:08Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "ee4bb12b-5d09-4b75-99e6-eb6df7655555", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-07-08T19:26:08Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "ee4bb12b-5d09-4b75-99e6-eb6df7655555", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-07-08T19:26:08Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "ee4bb12b-5d09-4b75-99e6-eb6df7655555", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-07-08T19:26:16Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "cd5c1e07-6741-4a9c-8e3a-4ccb731a3ca6", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-07-08T19:26:16Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "cd5c1e07-6741-4a9c-8e3a-4ccb731a3ca6", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-07-08T19:26:16Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "cd5c1e07-6741-4a9c-8e3a-4ccb731a3ca6", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-07-08T19:26:21Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "06a255e0-e851-4153-becf-88f99afcbc93", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-07-08T19:26:21Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "06a255e0-e851-4153-becf-88f99afcbc93", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-07-08T19:26:21Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "06a255e0-e851-4153-becf-88f99afcbc93", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-07-08T19:26:24Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-ait-wrong-c74617bb"}, "namespace": "e2e-ait-wrong-c74617bb", "name": "openshift-service-ca.crt", "reconcileID": "735a61f4-19c9-4c7f-bcc1-9b7a0e4ceee0", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-ait-wrong-c74617bb"} 2026-07-08T19:26:24Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-ait-wrong-c74617bb"}, "namespace": "e2e-ait-wrong-c74617bb", "name": "odh-trusted-ca-bundle", "reconcileID": "e9c9236b-0974-4146-ad73-2565ee74f94b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-ait-wrong-c74617bb"} 2026-07-08T19:26:24Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-ait-wrong-c74617bb"}, "namespace": "e2e-ait-wrong-c74617bb", "name": "openshift-service-ca.crt", "reconcileID": "103d21c4-d148-4d6c-aaae-7fdec03e48f4", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-ait-wrong-c74617bb", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:26:24Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-ait-wrong-c74617bb"}, "namespace": "e2e-ait-wrong-c74617bb", "name": "openshift-service-ca.crt", "reconcileID": "103d21c4-d148-4d6c-aaae-7fdec03e48f4", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-ait-wrong-c74617bb"} 2026-07-08T19:26:24Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-ait-wrong-c74617bb"}, "namespace": "e2e-ait-wrong-c74617bb", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fffde925-517c-4e6d-8180-63a5886aceda", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-ait-wrong-c74617bb"} 2026-07-08T19:26:24Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-ait-wrong-c74617bb"}, "namespace": "e2e-ait-wrong-c74617bb", "name": "openshift-service-ca.crt", "reconcileID": "279f8a0b-408c-46b6-be2c-56655117896c", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-ait-wrong-c74617bb"} 2026-07-08T19:26:24Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-ait-wrong-c74617bb"}, "namespace": "e2e-ait-wrong-c74617bb", "name": "odh-trusted-ca-bundle", "reconcileID": "c794911e-4966-4b25-805c-1b182d28e5e4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-ait-wrong-c74617bb", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:26:24Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-ait-wrong-c74617bb"}, "namespace": "e2e-ait-wrong-c74617bb", "name": "odh-trusted-ca-bundle", "reconcileID": "c794911e-4966-4b25-805c-1b182d28e5e4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-ait-wrong-c74617bb"} 2026-07-08T19:26:24Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-ait-wrong-c74617bb"}, "namespace": "e2e-ait-wrong-c74617bb", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f4ac0acb-0afb-4271-a43e-d480b651eaf5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-ait-wrong-c74617bb"} 2026-07-08T19:26:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-ait-wrong-c74617bb"}, "namespace": "e2e-ait-wrong-c74617bb", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "74b357f2-2a09-4c6d-9e00-71518eb24594", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-ait-wrong-c74617bb"} 2026-07-08T19:26:30Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-ait-wrong-c74617bb"}, "namespace": "e2e-ait-wrong-c74617bb", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "74b357f2-2a09-4c6d-9e00-71518eb24594", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-ait-wrong-c74617bb", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:26:30Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-ait-wrong-c74617bb"}, "namespace": "e2e-ait-wrong-c74617bb", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "74b357f2-2a09-4c6d-9e00-71518eb24594", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-ait-wrong-c74617bb because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T19:26:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-ait-wrong-c74617bb"}, "namespace": "e2e-ait-wrong-c74617bb", "name": "odh-trusted-ca-bundle", "reconcileID": "a37843ab-0773-4ca8-9742-751e48cdcd68", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-ait-wrong-c74617bb"} 2026-07-08T19:26:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-ait-wrong-c74617bb"}, "namespace": "e2e-ait-wrong-c74617bb", "name": "odh-trusted-ca-bundle", "reconcileID": "a37843ab-0773-4ca8-9742-751e48cdcd68", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-ait-wrong-c74617bb"} 2026-07-08T19:26:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-ait-wrong-c74617bb"}, "namespace": "e2e-ait-wrong-c74617bb", "name": "openshift-service-ca.crt", "reconcileID": "1bf385bb-2185-444c-badc-cdc1aa774f60", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-ait-wrong-c74617bb"} 2026-07-08T19:26:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-ait-wrong-c74617bb"}, "namespace": "e2e-ait-wrong-c74617bb", "name": "openshift-service-ca.crt", "reconcileID": "1bf385bb-2185-444c-badc-cdc1aa774f60", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-ait-wrong-c74617bb"} 2026-07-08T19:26:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-ait-wrong-c74617bb"}, "namespace": "e2e-ait-wrong-c74617bb", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9e4537b2-a394-4a34-bf64-87cc92b8311b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-ait-wrong-c74617bb"} 2026-07-08T19:26:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-ait-wrong-c74617bb"}, "namespace": "e2e-ait-wrong-c74617bb", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9e4537b2-a394-4a34-bf64-87cc92b8311b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-ait-wrong-c74617bb"} 2026-07-08T19:26:31Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-16aad909","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-16aad909", "reconcileID": "3e946e52-f050-440d-a995-c573be238fd0", "Gateway": "e2e-ait-16aad909", "namespace": "openshift-ingress"} 2026-07-08T19:26:31Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-16aad909","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-16aad909", "reconcileID": "b21eac38-8989-4323-a6d7-27bddd9e5fcb", "Gateway": "e2e-ait-16aad909", "namespace": "openshift-ingress"} 2026-07-08T19:26:31Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-16aad909","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-16aad909", "reconcileID": "8138b894-36de-43ff-98ac-6953e182d820", "Gateway": "e2e-ait-16aad909", "namespace": "openshift-ingress"} 2026-07-08T19:26:31Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-16aad909","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-16aad909", "reconcileID": "690ae0ae-392d-44cc-aa84-82245aad2cc7", "Gateway": "e2e-ait-16aad909", "namespace": "openshift-ingress"} 2026-07-08T19:26:31Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-16aad909","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-16aad909", "reconcileID": "3291f948-4498-41d3-8fd3-33f1e631eb81", "Gateway": "e2e-ait-16aad909", "namespace": "openshift-ingress"} 2026-07-08T19:26:32Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-16aad909","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-16aad909", "reconcileID": "bf99c7e3-6c49-49fa-a67f-2d256449d421", "Gateway": "e2e-ait-16aad909", "namespace": "openshift-ingress"} 2026-07-08T19:26:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-16aad909"}, "namespace": "ai-tenant-e2e-ait-16aad909", "name": "openshift-service-ca.crt", "reconcileID": "383fc44a-34f7-475f-8571-e19988d1abc9", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-16aad909"} 2026-07-08T19:26:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-16aad909"}, "namespace": "ai-tenant-e2e-ait-16aad909", "name": "odh-trusted-ca-bundle", "reconcileID": "51e709a9-a91c-42d7-8aff-7f3d5eb20fea", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-16aad909"} 2026-07-08T19:26:32Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-16aad909"}, "namespace": "ai-tenant-e2e-ait-16aad909", "name": "openshift-service-ca.crt", "reconcileID": "822762f2-b1f8-46a8-8a35-9919b6790f33", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-16aad909", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:26:32Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-16aad909"}, "namespace": "ai-tenant-e2e-ait-16aad909", "name": "openshift-service-ca.crt", "reconcileID": "822762f2-b1f8-46a8-8a35-9919b6790f33", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-16aad909"} 2026-07-08T19:26:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-16aad909"}, "namespace": "ai-tenant-e2e-ait-16aad909", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5631425a-70ca-4b5f-8f6a-97ab73387936", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-16aad909"} 2026-07-08T19:26:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-16aad909"}, "namespace": "ai-tenant-e2e-ait-16aad909", "name": "openshift-service-ca.crt", "reconcileID": "0c9345da-6f9b-4cc8-a2e8-f906e3de9682", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-16aad909"} 2026-07-08T19:26:32Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-16aad909"}, "namespace": "ai-tenant-e2e-ait-16aad909", "name": "odh-trusted-ca-bundle", "reconcileID": "d60c20a1-e5cc-46b4-ba81-35b3a6f10baf", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-16aad909", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:26:32Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-16aad909"}, "namespace": "ai-tenant-e2e-ait-16aad909", "name": "odh-trusted-ca-bundle", "reconcileID": "d60c20a1-e5cc-46b4-ba81-35b3a6f10baf", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-16aad909"} 2026-07-08T19:26:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-16aad909"}, "namespace": "ai-tenant-e2e-ait-16aad909", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fb716768-4460-41d5-99b9-170ce0f7593a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-16aad909"} 2026-07-08T19:26:33Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "59f565d0-ebee-4fb1-a2b6-34a61186834d", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T19:26:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-16aad909","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-16aad909", "reconcileID": "f03ae6fe-75cf-4875-9411-818df6ec562c", "Gateway": "e2e-ait-16aad909", "namespace": "openshift-ingress"} 2026-07-08T19:26:42Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-16aad909","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-16aad909", "reconcileID": "70d5b5bf-cc0c-4af9-9fab-5aac7d338f94", "Gateway": "e2e-ait-16aad909", "namespace": "openshift-ingress"} 2026-07-08T19:27:43Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-16aad909"}, "namespace": "ai-tenant-e2e-ait-16aad909", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f4e289b8-23bc-4010-9847-acad346bfbdc", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-16aad909"} 2026-07-08T19:27:43Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-16aad909"}, "namespace": "ai-tenant-e2e-ait-16aad909", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f4e289b8-23bc-4010-9847-acad346bfbdc", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-16aad909", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:27:43Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-16aad909"}, "namespace": "ai-tenant-e2e-ait-16aad909", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f4e289b8-23bc-4010-9847-acad346bfbdc", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ait-16aad909 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T19:27:43Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-16aad909"}, "namespace": "ai-tenant-e2e-ait-16aad909", "name": "odh-trusted-ca-bundle", "reconcileID": "1ba6f26d-3087-4dc7-877b-83cc12e6a2de", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-16aad909"} 2026-07-08T19:27:43Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-16aad909"}, "namespace": "ai-tenant-e2e-ait-16aad909", "name": "odh-trusted-ca-bundle", "reconcileID": "1ba6f26d-3087-4dc7-877b-83cc12e6a2de", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-16aad909"} 2026-07-08T19:27:43Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-16aad909"}, "namespace": "ai-tenant-e2e-ait-16aad909", "name": "openshift-service-ca.crt", "reconcileID": "fd83af97-17f5-47ca-ae7a-35d57ffce2b0", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-16aad909"} 2026-07-08T19:27:43Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-16aad909"}, "namespace": "ai-tenant-e2e-ait-16aad909", "name": "openshift-service-ca.crt", "reconcileID": "fd83af97-17f5-47ca-ae7a-35d57ffce2b0", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-16aad909"} 2026-07-08T19:27:43Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-16aad909"}, "namespace": "ai-tenant-e2e-ait-16aad909", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f97ad81c-f9ed-4931-b01a-0804805d4d85", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-16aad909"} 2026-07-08T19:27:43Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-16aad909"}, "namespace": "ai-tenant-e2e-ait-16aad909", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f97ad81c-f9ed-4931-b01a-0804805d4d85", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-16aad909"} 2026-07-08T19:27:50Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-693d5a39","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-693d5a39", "reconcileID": "c36691ce-46fd-49b1-863c-a67d7f9ad3f1", "Gateway": "e2e-ait-693d5a39", "namespace": "openshift-ingress"} 2026-07-08T19:27:50Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-693d5a39","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-693d5a39", "reconcileID": "80d7c44f-8b61-4ea3-bbf2-4ec95c1c7ecf", "Gateway": "e2e-ait-693d5a39", "namespace": "openshift-ingress"} 2026-07-08T19:27:50Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-693d5a39","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-693d5a39", "reconcileID": "0374f38e-6551-4852-8645-4be863ed4340", "Gateway": "e2e-ait-693d5a39", "namespace": "openshift-ingress"} 2026-07-08T19:27:50Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-693d5a39","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-693d5a39", "reconcileID": "089add1a-333a-4c82-9c4d-2cc73072002d", "Gateway": "e2e-ait-693d5a39", "namespace": "openshift-ingress"} 2026-07-08T19:27:50Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-693d5a39","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-693d5a39", "reconcileID": "74f7e8a9-efb6-429b-ab28-3e8934cdfd11", "Gateway": "e2e-ait-693d5a39", "namespace": "openshift-ingress"} 2026-07-08T19:27:51Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-693d5a39","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-693d5a39", "reconcileID": "b8f08c62-5985-4f35-b872-39857ff90533", "Gateway": "e2e-ait-693d5a39", "namespace": "openshift-ingress"} 2026-07-08T19:27:53Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-693d5a39","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-693d5a39", "reconcileID": "b92fb7fb-a820-40ad-8fc3-5a52b905c1b4", "Gateway": "e2e-ait-693d5a39", "namespace": "openshift-ingress"} 2026-07-08T19:29:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "5ba721f4-5d13-4d68-9c96-76606305cce7", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T19:30:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-3e4bf33f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-3e4bf33f", "reconcileID": "625b5dcc-dfdb-4da7-9f2d-9145e4c8a4f6", "Gateway": "e2e-derive-3e4bf33f", "namespace": "openshift-ingress"} 2026-07-08T19:30:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-3e4bf33f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-3e4bf33f", "reconcileID": "10791142-0079-45fe-8f15-61fb72203f55", "Gateway": "e2e-derive-3e4bf33f", "namespace": "openshift-ingress"} 2026-07-08T19:30:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-3e4bf33f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-3e4bf33f", "reconcileID": "27f691e6-10c0-4079-9f8d-a1f06d5f3657", "Gateway": "e2e-derive-3e4bf33f", "namespace": "openshift-ingress"} 2026-07-08T19:30:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-3e4bf33f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-3e4bf33f", "reconcileID": "214632e2-9dbb-4f95-9cf5-eaa4226c490b", "Gateway": "e2e-derive-3e4bf33f", "namespace": "openshift-ingress"} 2026-07-08T19:30:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-3e4bf33f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-3e4bf33f", "reconcileID": "f0ab311f-d8c0-4bff-81a4-c982c15d53d6", "Gateway": "e2e-derive-3e4bf33f", "namespace": "openshift-ingress"} 2026-07-08T19:30:56Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-3e4bf33f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-3e4bf33f", "reconcileID": "030bec6f-0516-44d9-b5c2-6a65996b142c", "Gateway": "e2e-derive-3e4bf33f", "namespace": "openshift-ingress"} 2026-07-08T19:31:10Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-3e4bf33f","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-3e4bf33f", "reconcileID": "355d6c51-e3fb-4c5c-9731-88332fabf515", "Gateway": "e2e-derive-3e4bf33f", "namespace": "openshift-ingress"} 2026-07-08T19:32:17Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-08T19:32:17Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-08T19:32:17Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-08T19:32:17Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-08T19:32:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-6b4cd5d2"}, "namespace": "ai-tenant-e2e-mt-6b4cd5d2", "name": "openshift-service-ca.crt", "reconcileID": "9f15218b-5f39-4be2-bc79-c32d9ef34b7f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-6b4cd5d2"} 2026-07-08T19:32:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-6b4cd5d2"}, "namespace": "ai-tenant-e2e-mt-6b4cd5d2", "name": "odh-trusted-ca-bundle", "reconcileID": "0c16a9fb-c63c-4d52-b52c-02a22d6f7338", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-6b4cd5d2"} 2026-07-08T19:32:59Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-6b4cd5d2"}, "namespace": "ai-tenant-e2e-mt-6b4cd5d2", "name": "openshift-service-ca.crt", "reconcileID": "78326480-92d5-442b-b29a-baf341d8d875", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-6b4cd5d2", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:32:59Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-6b4cd5d2"}, "namespace": "ai-tenant-e2e-mt-6b4cd5d2", "name": "openshift-service-ca.crt", "reconcileID": "78326480-92d5-442b-b29a-baf341d8d875", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-6b4cd5d2"} 2026-07-08T19:32:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-6b4cd5d2"}, "namespace": "ai-tenant-e2e-mt-6b4cd5d2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7c777ba3-639b-4411-885e-246ec59d1d92", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-6b4cd5d2"} 2026-07-08T19:32:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-6b4cd5d2"}, "namespace": "ai-tenant-e2e-mt-6b4cd5d2", "name": "openshift-service-ca.crt", "reconcileID": "7afa5752-97ec-4b84-bccc-b2316d027fe5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-6b4cd5d2"} 2026-07-08T19:32:59Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-6b4cd5d2"}, "namespace": "ai-tenant-e2e-mt-6b4cd5d2", "name": "odh-trusted-ca-bundle", "reconcileID": "98fd2f92-8af0-4012-89db-60a84af9a8cc", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-6b4cd5d2", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:32:59Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-6b4cd5d2"}, "namespace": "ai-tenant-e2e-mt-6b4cd5d2", "name": "odh-trusted-ca-bundle", "reconcileID": "98fd2f92-8af0-4012-89db-60a84af9a8cc", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-6b4cd5d2"} 2026-07-08T19:32:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-6b4cd5d2"}, "namespace": "ai-tenant-e2e-mt-6b4cd5d2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fb861104-9194-46f8-80eb-d2936d1f5c3e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-6b4cd5d2"} 2026-07-08T19:33:06Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-6b4cd5d2"}, "namespace": "ai-tenant-e2e-mt-6b4cd5d2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5343b4b8-02c5-495d-a3c6-75948055d992", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-6b4cd5d2"} 2026-07-08T19:33:06Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-6b4cd5d2"}, "namespace": "ai-tenant-e2e-mt-6b4cd5d2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5343b4b8-02c5-495d-a3c6-75948055d992", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-6b4cd5d2", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:33:06Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-6b4cd5d2"}, "namespace": "ai-tenant-e2e-mt-6b4cd5d2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5343b4b8-02c5-495d-a3c6-75948055d992", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-6b4cd5d2 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T19:33:06Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-6b4cd5d2"}, "namespace": "ai-tenant-e2e-mt-6b4cd5d2", "name": "odh-trusted-ca-bundle", "reconcileID": "67b3e9ca-ccb5-48d6-b0b9-7602862635f1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-6b4cd5d2"} 2026-07-08T19:33:06Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-6b4cd5d2"}, "namespace": "ai-tenant-e2e-mt-6b4cd5d2", "name": "odh-trusted-ca-bundle", "reconcileID": "67b3e9ca-ccb5-48d6-b0b9-7602862635f1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-6b4cd5d2"} 2026-07-08T19:33:06Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-6b4cd5d2"}, "namespace": "ai-tenant-e2e-mt-6b4cd5d2", "name": "openshift-service-ca.crt", "reconcileID": "601a4fd5-e404-499d-b965-a00732ed8553", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-6b4cd5d2"} 2026-07-08T19:33:06Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-6b4cd5d2"}, "namespace": "ai-tenant-e2e-mt-6b4cd5d2", "name": "openshift-service-ca.crt", "reconcileID": "601a4fd5-e404-499d-b965-a00732ed8553", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-6b4cd5d2"} 2026-07-08T19:33:06Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-6b4cd5d2"}, "namespace": "ai-tenant-e2e-mt-6b4cd5d2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6324ec2b-0637-457d-a493-10266724235d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-6b4cd5d2"} 2026-07-08T19:33:06Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-6b4cd5d2"}, "namespace": "ai-tenant-e2e-mt-6b4cd5d2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6324ec2b-0637-457d-a493-10266724235d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-6b4cd5d2"} 2026-07-08T19:33:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c6461cdb"}, "namespace": "ai-tenant-e2e-mt-c6461cdb", "name": "openshift-service-ca.crt", "reconcileID": "2f95fcfa-0250-473a-a1f4-2d92dd7a74e7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c6461cdb"} 2026-07-08T19:33:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c6461cdb"}, "namespace": "ai-tenant-e2e-mt-c6461cdb", "name": "odh-trusted-ca-bundle", "reconcileID": "664880f8-be2a-4b97-9c2b-0b8e29a975d9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c6461cdb"} 2026-07-08T19:33:07Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c6461cdb"}, "namespace": "ai-tenant-e2e-mt-c6461cdb", "name": "openshift-service-ca.crt", "reconcileID": "04ad206a-51a7-4346-8afe-6c2c355fef6b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c6461cdb", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:33:07Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c6461cdb"}, "namespace": "ai-tenant-e2e-mt-c6461cdb", "name": "openshift-service-ca.crt", "reconcileID": "04ad206a-51a7-4346-8afe-6c2c355fef6b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c6461cdb"} 2026-07-08T19:33:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c6461cdb"}, "namespace": "ai-tenant-e2e-mt-c6461cdb", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "946c0e4d-41f5-42b6-9d1a-6b292238a8aa", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c6461cdb"} 2026-07-08T19:33:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c6461cdb"}, "namespace": "ai-tenant-e2e-mt-c6461cdb", "name": "openshift-service-ca.crt", "reconcileID": "4406f735-5537-4b52-888a-ffb1c5248374", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c6461cdb"} 2026-07-08T19:33:07Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c6461cdb"}, "namespace": "ai-tenant-e2e-mt-c6461cdb", "name": "odh-trusted-ca-bundle", "reconcileID": "86cc04f2-8961-433a-a9da-d05cf333750d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c6461cdb", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:33:07Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c6461cdb"}, "namespace": "ai-tenant-e2e-mt-c6461cdb", "name": "odh-trusted-ca-bundle", "reconcileID": "86cc04f2-8961-433a-a9da-d05cf333750d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c6461cdb"} 2026-07-08T19:33:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c6461cdb"}, "namespace": "ai-tenant-e2e-mt-c6461cdb", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "84195ac3-a98e-46f1-9e5f-5626731a6abc", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c6461cdb"} 2026-07-08T19:33:39Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c6461cdb"}, "namespace": "ai-tenant-e2e-mt-c6461cdb", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "aab7b388-68b3-49eb-aae4-2d4a313345c8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c6461cdb"} 2026-07-08T19:33:39Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c6461cdb"}, "namespace": "ai-tenant-e2e-mt-c6461cdb", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "aab7b388-68b3-49eb-aae4-2d4a313345c8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c6461cdb", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:33:39Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c6461cdb"}, "namespace": "ai-tenant-e2e-mt-c6461cdb", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "aab7b388-68b3-49eb-aae4-2d4a313345c8", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-c6461cdb because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T19:33:39Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c6461cdb"}, "namespace": "ai-tenant-e2e-mt-c6461cdb", "name": "odh-trusted-ca-bundle", "reconcileID": "63626d01-618d-420d-b1f7-af7ae388c5cc", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c6461cdb"} 2026-07-08T19:33:39Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-c6461cdb"}, "namespace": "ai-tenant-e2e-mt-c6461cdb", "name": "odh-trusted-ca-bundle", "reconcileID": "63626d01-618d-420d-b1f7-af7ae388c5cc", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-c6461cdb"} 2026-07-08T19:33:39Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c6461cdb"}, "namespace": "ai-tenant-e2e-mt-c6461cdb", "name": "openshift-service-ca.crt", "reconcileID": "e703e9e6-45b8-4fdd-aad8-1b7ff1296b00", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c6461cdb"} 2026-07-08T19:33:39Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-c6461cdb"}, "namespace": "ai-tenant-e2e-mt-c6461cdb", "name": "openshift-service-ca.crt", "reconcileID": "e703e9e6-45b8-4fdd-aad8-1b7ff1296b00", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-c6461cdb"} 2026-07-08T19:33:39Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c6461cdb"}, "namespace": "ai-tenant-e2e-mt-c6461cdb", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d7b79094-ca68-4c95-88ff-86144a7071e6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c6461cdb"} 2026-07-08T19:33:39Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-c6461cdb"}, "namespace": "ai-tenant-e2e-mt-c6461cdb", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d7b79094-ca68-4c95-88ff-86144a7071e6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-c6461cdb"} 2026-07-08T19:33:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-f35e3045"}, "namespace": "e2e-unlabeled-f35e3045", "name": "openshift-service-ca.crt", "reconcileID": "9243fa2e-681b-46d6-b10b-27d9b1bb33d2", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-f35e3045"} 2026-07-08T19:33:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-f35e3045"}, "namespace": "e2e-unlabeled-f35e3045", "name": "odh-trusted-ca-bundle", "reconcileID": "c92ce244-9368-4023-880b-6a38e93f8850", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-f35e3045"} 2026-07-08T19:33:40Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-f35e3045"}, "namespace": "e2e-unlabeled-f35e3045", "name": "openshift-service-ca.crt", "reconcileID": "26e6911b-0824-4f36-a647-282565d37363", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-f35e3045", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:33:40Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-f35e3045"}, "namespace": "e2e-unlabeled-f35e3045", "name": "openshift-service-ca.crt", "reconcileID": "26e6911b-0824-4f36-a647-282565d37363", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-f35e3045"} 2026-07-08T19:33:40Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-f35e3045"}, "namespace": "e2e-unlabeled-f35e3045", "name": "openshift-service-ca.crt", "reconcileID": "f2c0d8df-af3b-4c43-9867-9565d8afa590", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-f35e3045", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:33:40Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-f35e3045"}, "namespace": "e2e-unlabeled-f35e3045", "name": "openshift-service-ca.crt", "reconcileID": "f2c0d8df-af3b-4c43-9867-9565d8afa590", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T19:33:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-f35e3045"}, "namespace": "e2e-unlabeled-f35e3045", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "354fdf00-f3b2-4b63-8492-46603396480b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-f35e3045"} 2026-07-08T19:33:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-f35e3045"}, "namespace": "e2e-unlabeled-f35e3045", "name": "openshift-service-ca.crt", "reconcileID": "dd018828-e7b0-4606-bd7a-384a28f04c27", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-f35e3045"} 2026-07-08T19:33:40Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-f35e3045"}, "namespace": "e2e-unlabeled-f35e3045", "name": "odh-trusted-ca-bundle", "reconcileID": "801340f5-12d7-4891-b8e2-6ed3fcabc3f1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-f35e3045", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:33:40Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-f35e3045"}, "namespace": "e2e-unlabeled-f35e3045", "name": "odh-trusted-ca-bundle", "reconcileID": "801340f5-12d7-4891-b8e2-6ed3fcabc3f1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-f35e3045"} 2026-07-08T19:33:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-f35e3045"}, "namespace": "e2e-unlabeled-f35e3045", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6976062d-8458-4cfe-84bb-bd42cf0d471a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-f35e3045"} 2026-07-08T19:34:01Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-f35e3045"}, "namespace": "e2e-unlabeled-f35e3045", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6f473d5b-6822-4f82-8cc6-7cd12cacf3c2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-f35e3045"} 2026-07-08T19:34:01Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-f35e3045"}, "namespace": "e2e-unlabeled-f35e3045", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6f473d5b-6822-4f82-8cc6-7cd12cacf3c2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-f35e3045", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:34:01Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-f35e3045"}, "namespace": "e2e-unlabeled-f35e3045", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6f473d5b-6822-4f82-8cc6-7cd12cacf3c2", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-unlabeled-f35e3045 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T19:34:01Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-f35e3045"}, "namespace": "e2e-unlabeled-f35e3045", "name": "odh-trusted-ca-bundle", "reconcileID": "0c75c083-058c-4f18-a840-8f2dba4c6ed2", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-f35e3045"} 2026-07-08T19:34:01Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-f35e3045"}, "namespace": "e2e-unlabeled-f35e3045", "name": "odh-trusted-ca-bundle", "reconcileID": "0c75c083-058c-4f18-a840-8f2dba4c6ed2", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-f35e3045"} 2026-07-08T19:34:01Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-f35e3045"}, "namespace": "e2e-unlabeled-f35e3045", "name": "openshift-service-ca.crt", "reconcileID": "666b53d6-d0ee-4a95-81c6-c604bb98c342", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-f35e3045"} 2026-07-08T19:34:01Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-f35e3045"}, "namespace": "e2e-unlabeled-f35e3045", "name": "openshift-service-ca.crt", "reconcileID": "666b53d6-d0ee-4a95-81c6-c604bb98c342", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-f35e3045"} 2026-07-08T19:34:01Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-f35e3045"}, "namespace": "e2e-unlabeled-f35e3045", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e5b278a2-0cb9-4051-a6d1-effa36d98cdb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-f35e3045"} 2026-07-08T19:34:01Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-f35e3045"}, "namespace": "e2e-unlabeled-f35e3045", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e5b278a2-0cb9-4051-a6d1-effa36d98cdb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-f35e3045"} 2026-07-08T19:34:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-23bf7596"}, "namespace": "ai-tenant-e2e-mt-23bf7596", "name": "openshift-service-ca.crt", "reconcileID": "d396551c-311c-487b-a897-a2367fc81eed", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-23bf7596"} 2026-07-08T19:34:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-23bf7596"}, "namespace": "ai-tenant-e2e-mt-23bf7596", "name": "odh-trusted-ca-bundle", "reconcileID": "f79e63eb-6900-423c-8947-c266d654c74f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-23bf7596"} 2026-07-08T19:34:02Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-23bf7596"}, "namespace": "ai-tenant-e2e-mt-23bf7596", "name": "openshift-service-ca.crt", "reconcileID": "aea42143-9d71-4782-995e-92b66407fbed", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-23bf7596", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:34:02Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-23bf7596"}, "namespace": "ai-tenant-e2e-mt-23bf7596", "name": "openshift-service-ca.crt", "reconcileID": "aea42143-9d71-4782-995e-92b66407fbed", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-23bf7596"} 2026-07-08T19:34:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-23bf7596"}, "namespace": "ai-tenant-e2e-mt-23bf7596", "name": "openshift-service-ca.crt", "reconcileID": "549317c4-414b-4546-bcb6-a0d65c22a1d3", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-23bf7596"} 2026-07-08T19:34:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-23bf7596"}, "namespace": "ai-tenant-e2e-mt-23bf7596", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9e88e4b8-beb3-48aa-8478-b67c408e1e1e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-23bf7596"} 2026-07-08T19:34:02Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-23bf7596"}, "namespace": "ai-tenant-e2e-mt-23bf7596", "name": "odh-trusted-ca-bundle", "reconcileID": "cd2f0172-2b41-4e6e-882d-8ae4797067e0", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-23bf7596", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:34:02Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-23bf7596"}, "namespace": "ai-tenant-e2e-mt-23bf7596", "name": "odh-trusted-ca-bundle", "reconcileID": "cd2f0172-2b41-4e6e-882d-8ae4797067e0", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-23bf7596"} 2026-07-08T19:34:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-23bf7596"}, "namespace": "ai-tenant-e2e-mt-23bf7596", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "715d9ee2-eedc-46ad-932f-f966142ff51b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-23bf7596"} 2026-07-08T19:34:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-23bf7596"}, "namespace": "ai-tenant-e2e-mt-23bf7596", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f96afea5-5cfc-4469-9e20-e91fd54bda9e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-23bf7596"} 2026-07-08T19:34:19Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-23bf7596"}, "namespace": "ai-tenant-e2e-mt-23bf7596", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f96afea5-5cfc-4469-9e20-e91fd54bda9e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-23bf7596", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:34:19Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-23bf7596"}, "namespace": "ai-tenant-e2e-mt-23bf7596", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f96afea5-5cfc-4469-9e20-e91fd54bda9e", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-23bf7596 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T19:34:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-23bf7596"}, "namespace": "ai-tenant-e2e-mt-23bf7596", "name": "odh-trusted-ca-bundle", "reconcileID": "740292af-17bf-4ab1-b663-c4d51cccc801", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-23bf7596"} 2026-07-08T19:34:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-23bf7596"}, "namespace": "ai-tenant-e2e-mt-23bf7596", "name": "odh-trusted-ca-bundle", "reconcileID": "740292af-17bf-4ab1-b663-c4d51cccc801", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-23bf7596"} 2026-07-08T19:34:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-23bf7596"}, "namespace": "ai-tenant-e2e-mt-23bf7596", "name": "openshift-service-ca.crt", "reconcileID": "1035919f-0e77-486b-8338-8ba8d8d77e9e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-23bf7596"} 2026-07-08T19:34:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-23bf7596"}, "namespace": "ai-tenant-e2e-mt-23bf7596", "name": "openshift-service-ca.crt", "reconcileID": "1035919f-0e77-486b-8338-8ba8d8d77e9e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-23bf7596"} 2026-07-08T19:34:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-23bf7596"}, "namespace": "ai-tenant-e2e-mt-23bf7596", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d5ed3bf9-0595-40d4-bb8e-8eaf63226672", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-23bf7596"} 2026-07-08T19:34:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-23bf7596"}, "namespace": "ai-tenant-e2e-mt-23bf7596", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d5ed3bf9-0595-40d4-bb8e-8eaf63226672", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-23bf7596"} 2026-07-08T19:34:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-3347a355"}, "namespace": "ai-tenant-e2e-mt-3347a355", "name": "openshift-service-ca.crt", "reconcileID": "c5485bdb-cea4-4cac-91d9-21d40293b862", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-3347a355"} 2026-07-08T19:34:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-3347a355"}, "namespace": "ai-tenant-e2e-mt-3347a355", "name": "odh-trusted-ca-bundle", "reconcileID": "e0a4544d-f969-4738-91b5-3a8b41c12caa", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-3347a355"} 2026-07-08T19:34:19Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-3347a355"}, "namespace": "ai-tenant-e2e-mt-3347a355", "name": "openshift-service-ca.crt", "reconcileID": "e6588fab-1c1b-4f0c-8bbb-255388abcc00", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-3347a355", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:34:19Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-3347a355"}, "namespace": "ai-tenant-e2e-mt-3347a355", "name": "openshift-service-ca.crt", "reconcileID": "e6588fab-1c1b-4f0c-8bbb-255388abcc00", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-3347a355"} 2026-07-08T19:34:19Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-3347a355"}, "namespace": "ai-tenant-e2e-mt-3347a355", "name": "openshift-service-ca.crt", "reconcileID": "b50c3e19-6a92-4242-8fa3-672908c506c9", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-3347a355", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:34:19Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-3347a355"}, "namespace": "ai-tenant-e2e-mt-3347a355", "name": "openshift-service-ca.crt", "reconcileID": "b50c3e19-6a92-4242-8fa3-672908c506c9", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T19:34:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-3347a355"}, "namespace": "ai-tenant-e2e-mt-3347a355", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fc830564-1f58-4425-b389-87e727c419d3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-3347a355"} 2026-07-08T19:34:19Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-3347a355"}, "namespace": "ai-tenant-e2e-mt-3347a355", "name": "odh-trusted-ca-bundle", "reconcileID": "375dab8f-ceb1-4956-bd0c-f9ea9b575d7d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-3347a355", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:34:19Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-3347a355"}, "namespace": "ai-tenant-e2e-mt-3347a355", "name": "odh-trusted-ca-bundle", "reconcileID": "375dab8f-ceb1-4956-bd0c-f9ea9b575d7d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-3347a355"} 2026-07-08T19:34:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-3347a355"}, "namespace": "ai-tenant-e2e-mt-3347a355", "name": "openshift-service-ca.crt", "reconcileID": "bd38d634-0e79-4508-8acb-f7426fdf16a6", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-3347a355"} 2026-07-08T19:34:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-3347a355"}, "namespace": "ai-tenant-e2e-mt-3347a355", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c3a43035-5185-4811-857b-bf22524f6464", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-3347a355"} 2026-07-08T19:34:20Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-7e8ffaa2"}, "namespace": "ai-tenant-e2e-mt-7e8ffaa2", "name": "openshift-service-ca.crt", "reconcileID": "95931c07-7d80-4cf5-804f-8ae5e18c4035", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-7e8ffaa2"} 2026-07-08T19:34:20Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-7e8ffaa2"}, "namespace": "ai-tenant-e2e-mt-7e8ffaa2", "name": "odh-trusted-ca-bundle", "reconcileID": "47e0145a-c629-43e1-a90c-6b785473f807", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-7e8ffaa2"} 2026-07-08T19:34:20Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-7e8ffaa2"}, "namespace": "ai-tenant-e2e-mt-7e8ffaa2", "name": "openshift-service-ca.crt", "reconcileID": "73f648ca-fca7-4532-8470-59e89ac677e8", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-7e8ffaa2", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:34:20Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-7e8ffaa2"}, "namespace": "ai-tenant-e2e-mt-7e8ffaa2", "name": "openshift-service-ca.crt", "reconcileID": "73f648ca-fca7-4532-8470-59e89ac677e8", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-7e8ffaa2"} 2026-07-08T19:34:20Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-7e8ffaa2"}, "namespace": "ai-tenant-e2e-mt-7e8ffaa2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d5d3c63c-43a4-4f42-a925-7aa36a5c27e9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-7e8ffaa2"} 2026-07-08T19:34:20Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-7e8ffaa2"}, "namespace": "ai-tenant-e2e-mt-7e8ffaa2", "name": "openshift-service-ca.crt", "reconcileID": "89913fd6-c074-4a6e-a385-37715d1a24ec", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-7e8ffaa2"} 2026-07-08T19:34:20Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-7e8ffaa2"}, "namespace": "ai-tenant-e2e-mt-7e8ffaa2", "name": "odh-trusted-ca-bundle", "reconcileID": "408f8288-6004-4f0d-a6a5-34464c882bb9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-7e8ffaa2", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:34:21Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-7e8ffaa2"}, "namespace": "ai-tenant-e2e-mt-7e8ffaa2", "name": "odh-trusted-ca-bundle", "reconcileID": "408f8288-6004-4f0d-a6a5-34464c882bb9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-7e8ffaa2"} 2026-07-08T19:34:21Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-7e8ffaa2"}, "namespace": "ai-tenant-e2e-mt-7e8ffaa2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0cb1d24c-b7d6-4ae2-8009-6cc8a669b7ea", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-7e8ffaa2"} 2026-07-08T19:34:27Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-3347a355"}, "namespace": "ai-tenant-e2e-mt-3347a355", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cd05fa77-8606-4a52-9bb5-b4c2a2b220d7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-3347a355"} 2026-07-08T19:34:27Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-3347a355"}, "namespace": "ai-tenant-e2e-mt-3347a355", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cd05fa77-8606-4a52-9bb5-b4c2a2b220d7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-3347a355", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:34:27Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-3347a355"}, "namespace": "ai-tenant-e2e-mt-3347a355", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cd05fa77-8606-4a52-9bb5-b4c2a2b220d7", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-3347a355 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T19:34:27Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-3347a355"}, "namespace": "ai-tenant-e2e-mt-3347a355", "name": "odh-trusted-ca-bundle", "reconcileID": "9245b49b-6072-4d23-9118-d703de2a9c22", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-3347a355"} 2026-07-08T19:34:27Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-3347a355"}, "namespace": "ai-tenant-e2e-mt-3347a355", "name": "odh-trusted-ca-bundle", "reconcileID": "9245b49b-6072-4d23-9118-d703de2a9c22", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-3347a355"} 2026-07-08T19:34:27Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-3347a355"}, "namespace": "ai-tenant-e2e-mt-3347a355", "name": "openshift-service-ca.crt", "reconcileID": "b3859969-2e2a-41de-babd-27d16756cfd1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-3347a355"} 2026-07-08T19:34:27Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-3347a355"}, "namespace": "ai-tenant-e2e-mt-3347a355", "name": "openshift-service-ca.crt", "reconcileID": "b3859969-2e2a-41de-babd-27d16756cfd1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-3347a355"} 2026-07-08T19:34:27Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-3347a355"}, "namespace": "ai-tenant-e2e-mt-3347a355", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ec35b862-1cf4-413b-aa3d-2c5ffa006b3d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-3347a355"} 2026-07-08T19:34:27Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-3347a355"}, "namespace": "ai-tenant-e2e-mt-3347a355", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ec35b862-1cf4-413b-aa3d-2c5ffa006b3d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-3347a355"} 2026-07-08T19:34:35Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-7e8ffaa2"}, "namespace": "ai-tenant-e2e-mt-7e8ffaa2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d7035ffb-3124-4532-91f0-3fe0c278f3f2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-7e8ffaa2"} 2026-07-08T19:34:35Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-7e8ffaa2"}, "namespace": "ai-tenant-e2e-mt-7e8ffaa2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d7035ffb-3124-4532-91f0-3fe0c278f3f2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-7e8ffaa2", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:34:35Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-7e8ffaa2"}, "namespace": "ai-tenant-e2e-mt-7e8ffaa2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d7035ffb-3124-4532-91f0-3fe0c278f3f2", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-7e8ffaa2 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T19:34:35Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-7e8ffaa2"}, "namespace": "ai-tenant-e2e-mt-7e8ffaa2", "name": "odh-trusted-ca-bundle", "reconcileID": "44045208-892a-42f1-9d70-d1a49025ed6b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-7e8ffaa2"} 2026-07-08T19:34:35Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-7e8ffaa2"}, "namespace": "ai-tenant-e2e-mt-7e8ffaa2", "name": "odh-trusted-ca-bundle", "reconcileID": "44045208-892a-42f1-9d70-d1a49025ed6b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-7e8ffaa2"} 2026-07-08T19:34:35Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-7e8ffaa2"}, "namespace": "ai-tenant-e2e-mt-7e8ffaa2", "name": "openshift-service-ca.crt", "reconcileID": "4b3ccdad-da72-4872-bcd2-32b30c2eae6b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-7e8ffaa2"} 2026-07-08T19:34:35Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-7e8ffaa2"}, "namespace": "ai-tenant-e2e-mt-7e8ffaa2", "name": "openshift-service-ca.crt", "reconcileID": "4b3ccdad-da72-4872-bcd2-32b30c2eae6b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-7e8ffaa2"} 2026-07-08T19:34:35Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-7e8ffaa2"}, "namespace": "ai-tenant-e2e-mt-7e8ffaa2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b58def23-d490-4c0c-9962-89d04224d4c5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-7e8ffaa2"} 2026-07-08T19:34:35Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-7e8ffaa2"}, "namespace": "ai-tenant-e2e-mt-7e8ffaa2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b58def23-d490-4c0c-9962-89d04224d4c5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-7e8ffaa2"} 2026-07-08T19:34:36Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-cddeca0a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a", "reconcileID": "4b90b72e-a050-4d20-98bd-fac84afe58b9", "Gateway": "e2e-mt-cddeca0a", "namespace": "openshift-ingress"} 2026-07-08T19:34:36Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-cddeca0a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a", "reconcileID": "4b90b72e-a050-4d20-98bd-fac84afe58b9", "Gateway": "e2e-mt-cddeca0a", "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a-authn-ssl"} 2026-07-08T19:34:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-cddeca0a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a", "reconcileID": "4b90b72e-a050-4d20-98bd-fac84afe58b9", "Gateway": "e2e-mt-cddeca0a", "namespace": "openshift-ingress"} 2026-07-08T19:34:36Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-cddeca0a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a", "reconcileID": "90650422-9aa7-4d77-9960-991d64905ecd", "Gateway": "e2e-mt-cddeca0a", "namespace": "openshift-ingress"} 2026-07-08T19:34:36Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-cddeca0a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a", "reconcileID": "90650422-9aa7-4d77-9960-991d64905ecd", "Gateway": "e2e-mt-cddeca0a", "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a-authn-ssl"} 2026-07-08T19:34:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-cddeca0a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a", "reconcileID": "90650422-9aa7-4d77-9960-991d64905ecd", "Gateway": "e2e-mt-cddeca0a", "namespace": "openshift-ingress"} 2026-07-08T19:34:36Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-cddeca0a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a", "reconcileID": "a051fe39-0e12-456f-baf7-ba89a5c66c8e", "Gateway": "e2e-mt-cddeca0a", "namespace": "openshift-ingress"} 2026-07-08T19:34:36Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-cddeca0a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a", "reconcileID": "a051fe39-0e12-456f-baf7-ba89a5c66c8e", "Gateway": "e2e-mt-cddeca0a", "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a-authn-ssl"} 2026-07-08T19:34:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-cddeca0a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a", "reconcileID": "a051fe39-0e12-456f-baf7-ba89a5c66c8e", "Gateway": "e2e-mt-cddeca0a", "namespace": "openshift-ingress"} 2026-07-08T19:34:36Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-cddeca0a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a", "reconcileID": "cddf712c-9390-4c5d-94ff-f2d701c56934", "Gateway": "e2e-mt-cddeca0a", "namespace": "openshift-ingress"} 2026-07-08T19:34:36Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-cddeca0a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a", "reconcileID": "cddf712c-9390-4c5d-94ff-f2d701c56934", "Gateway": "e2e-mt-cddeca0a", "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a-authn-ssl"} 2026-07-08T19:34:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-cddeca0a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a", "reconcileID": "cddf712c-9390-4c5d-94ff-f2d701c56934", "Gateway": "e2e-mt-cddeca0a", "namespace": "openshift-ingress"} 2026-07-08T19:34:36Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-cddeca0a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a", "reconcileID": "8526891b-f56c-4489-8364-9bf7c759f871", "Gateway": "e2e-mt-cddeca0a", "namespace": "openshift-ingress"} 2026-07-08T19:34:36Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-cddeca0a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a", "reconcileID": "8526891b-f56c-4489-8364-9bf7c759f871", "Gateway": "e2e-mt-cddeca0a", "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a-authn-ssl"} 2026-07-08T19:34:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-cddeca0a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a", "reconcileID": "8526891b-f56c-4489-8364-9bf7c759f871", "Gateway": "e2e-mt-cddeca0a", "namespace": "openshift-ingress"} 2026-07-08T19:34:36Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-cddeca0a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a", "reconcileID": "dc136325-dfe5-415a-856a-d2bf49338787", "Gateway": "e2e-mt-cddeca0a", "namespace": "openshift-ingress"} 2026-07-08T19:34:36Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-cddeca0a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a", "reconcileID": "dc136325-dfe5-415a-856a-d2bf49338787", "Gateway": "e2e-mt-cddeca0a", "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a-authn-ssl"} 2026-07-08T19:34:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-cddeca0a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a", "reconcileID": "dc136325-dfe5-415a-856a-d2bf49338787", "Gateway": "e2e-mt-cddeca0a", "namespace": "openshift-ingress"} 2026-07-08T19:34:37Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-cddeca0a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a", "reconcileID": "4b278f35-633f-4358-93c3-011ab4e38ec3", "Gateway": "e2e-mt-cddeca0a", "namespace": "openshift-ingress"} 2026-07-08T19:34:37Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-cddeca0a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a", "reconcileID": "4b278f35-633f-4358-93c3-011ab4e38ec3", "Gateway": "e2e-mt-cddeca0a", "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a-authn-ssl"} 2026-07-08T19:34:37Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-cddeca0a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a", "reconcileID": "4b278f35-633f-4358-93c3-011ab4e38ec3", "Gateway": "e2e-mt-cddeca0a", "namespace": "openshift-ingress"} 2026-07-08T19:37:44Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-cddeca0a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-cddeca0a", "reconcileID": "6b28178d-f920-4853-a008-6127f9fe6841", "Gateway": "e2e-mt-cddeca0a", "namespace": "openshift-ingress"} 2026-07-08T19:37:44Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-df244b11"}, "namespace": "e2e-webhook-sub-df244b11", "name": "openshift-service-ca.crt", "reconcileID": "8e0c357c-06f5-41bb-b788-0bf9c991ee8d", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-df244b11"} 2026-07-08T19:37:44Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-df244b11"}, "namespace": "e2e-webhook-sub-df244b11", "name": "odh-trusted-ca-bundle", "reconcileID": "e4dcaf89-4fc0-4ee1-966f-444c31a35ade", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-df244b11"} 2026-07-08T19:37:44Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-df244b11"}, "namespace": "e2e-webhook-sub-df244b11", "name": "openshift-service-ca.crt", "reconcileID": "231094d1-31e9-4727-9550-cd81ddccd5e0", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-df244b11", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:37:44Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-df244b11"}, "namespace": "e2e-webhook-sub-df244b11", "name": "openshift-service-ca.crt", "reconcileID": "231094d1-31e9-4727-9550-cd81ddccd5e0", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-df244b11"} 2026-07-08T19:37:44Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-df244b11"}, "namespace": "e2e-webhook-sub-df244b11", "name": "openshift-service-ca.crt", "reconcileID": "6d1effc3-842e-4a6f-8e03-ffce645392d2", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-df244b11", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:37:44Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-df244b11"}, "namespace": "e2e-webhook-sub-df244b11", "name": "openshift-service-ca.crt", "reconcileID": "6d1effc3-842e-4a6f-8e03-ffce645392d2", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T19:37:44Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-df244b11"}, "namespace": "e2e-webhook-sub-df244b11", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ba35e172-2015-4113-9301-2f6655b17eb9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-df244b11"} 2026-07-08T19:37:44Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-df244b11"}, "namespace": "e2e-webhook-sub-df244b11", "name": "openshift-service-ca.crt", "reconcileID": "70f41e4d-c08a-4d2d-9823-7b600827f740", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-df244b11"} 2026-07-08T19:37:44Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-df244b11"}, "namespace": "e2e-webhook-sub-df244b11", "name": "odh-trusted-ca-bundle", "reconcileID": "5ed92e9b-ddce-4664-969a-0f221b488796", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-df244b11", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:37:44Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-df244b11"}, "namespace": "e2e-webhook-sub-df244b11", "name": "odh-trusted-ca-bundle", "reconcileID": "5ed92e9b-ddce-4664-969a-0f221b488796", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-df244b11"} 2026-07-08T19:37:44Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-df244b11"}, "namespace": "e2e-webhook-sub-df244b11", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c0349330-74e8-4b22-911f-5c8881e73e33", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-df244b11"} 2026-07-08T19:37:50Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-df244b11"}, "namespace": "e2e-webhook-sub-df244b11", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b52f8230-d751-481d-b0c7-add3991568b5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-df244b11"} 2026-07-08T19:37:50Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-df244b11"}, "namespace": "e2e-webhook-sub-df244b11", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b52f8230-d751-481d-b0c7-add3991568b5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-df244b11", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:37:50Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-df244b11"}, "namespace": "e2e-webhook-sub-df244b11", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b52f8230-d751-481d-b0c7-add3991568b5", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-sub-df244b11 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T19:37:50Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-df244b11"}, "namespace": "e2e-webhook-sub-df244b11", "name": "odh-trusted-ca-bundle", "reconcileID": "d579f1cf-18d2-4454-b99c-2d6fb33b5b51", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-df244b11"} 2026-07-08T19:37:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-df244b11"}, "namespace": "e2e-webhook-sub-df244b11", "name": "odh-trusted-ca-bundle", "reconcileID": "d579f1cf-18d2-4454-b99c-2d6fb33b5b51", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-df244b11"} 2026-07-08T19:37:50Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-df244b11"}, "namespace": "e2e-webhook-sub-df244b11", "name": "openshift-service-ca.crt", "reconcileID": "b72ee227-d933-4186-bcdc-51d9154678d8", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-df244b11"} 2026-07-08T19:37:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-df244b11"}, "namespace": "e2e-webhook-sub-df244b11", "name": "openshift-service-ca.crt", "reconcileID": "b72ee227-d933-4186-bcdc-51d9154678d8", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-df244b11"} 2026-07-08T19:37:50Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-df244b11"}, "namespace": "e2e-webhook-sub-df244b11", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6333a78a-ddee-416c-8b7b-4f7ddb14d41b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-df244b11"} 2026-07-08T19:37:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-df244b11"}, "namespace": "e2e-webhook-sub-df244b11", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6333a78a-ddee-416c-8b7b-4f7ddb14d41b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-df244b11"} 2026-07-08T19:37:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-c4c0d116"}, "namespace": "e2e-webhook-auth-c4c0d116", "name": "openshift-service-ca.crt", "reconcileID": "006c46af-c54e-4191-a531-1866204cd746", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-c4c0d116"} 2026-07-08T19:37:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-c4c0d116"}, "namespace": "e2e-webhook-auth-c4c0d116", "name": "odh-trusted-ca-bundle", "reconcileID": "d806f52a-68b3-4fc1-b5e8-be2be4868b07", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-c4c0d116"} 2026-07-08T19:37:50Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-c4c0d116"}, "namespace": "e2e-webhook-auth-c4c0d116", "name": "openshift-service-ca.crt", "reconcileID": "ad12fe6c-e27d-4bf0-b7a2-48d51dbbbe03", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-c4c0d116", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:37:50Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-c4c0d116"}, "namespace": "e2e-webhook-auth-c4c0d116", "name": "openshift-service-ca.crt", "reconcileID": "ad12fe6c-e27d-4bf0-b7a2-48d51dbbbe03", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-c4c0d116"} 2026-07-08T19:37:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-c4c0d116"}, "namespace": "e2e-webhook-auth-c4c0d116", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "97a3f380-103d-4135-898e-ac301ec6b7e7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-c4c0d116"} 2026-07-08T19:37:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-c4c0d116"}, "namespace": "e2e-webhook-auth-c4c0d116", "name": "openshift-service-ca.crt", "reconcileID": "11da8485-66fe-4242-8ba5-203a6e0c4917", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-c4c0d116"} 2026-07-08T19:37:50Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-c4c0d116"}, "namespace": "e2e-webhook-auth-c4c0d116", "name": "odh-trusted-ca-bundle", "reconcileID": "934365c5-a6f3-45fb-a778-1186121f2a86", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-c4c0d116", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:37:50Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-c4c0d116"}, "namespace": "e2e-webhook-auth-c4c0d116", "name": "odh-trusted-ca-bundle", "reconcileID": "934365c5-a6f3-45fb-a778-1186121f2a86", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-c4c0d116"} 2026-07-08T19:37:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-c4c0d116"}, "namespace": "e2e-webhook-auth-c4c0d116", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3b251ac1-3b0a-4a58-8d39-c303223449b3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-c4c0d116"} 2026-07-08T19:37:56Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-c4c0d116"}, "namespace": "e2e-webhook-auth-c4c0d116", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2980ba7e-1ceb-43ba-80c0-fd8f36657874", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-c4c0d116"} 2026-07-08T19:37:56Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-c4c0d116"}, "namespace": "e2e-webhook-auth-c4c0d116", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2980ba7e-1ceb-43ba-80c0-fd8f36657874", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-c4c0d116", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:37:56Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-c4c0d116"}, "namespace": "e2e-webhook-auth-c4c0d116", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2980ba7e-1ceb-43ba-80c0-fd8f36657874", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-auth-c4c0d116 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T19:37:56Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-c4c0d116"}, "namespace": "e2e-webhook-auth-c4c0d116", "name": "odh-trusted-ca-bundle", "reconcileID": "08bf4e0e-be54-46cc-906c-e9e7a8293612", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-c4c0d116"} 2026-07-08T19:37:56Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-c4c0d116"}, "namespace": "e2e-webhook-auth-c4c0d116", "name": "odh-trusted-ca-bundle", "reconcileID": "08bf4e0e-be54-46cc-906c-e9e7a8293612", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-c4c0d116"} 2026-07-08T19:37:56Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-c4c0d116"}, "namespace": "e2e-webhook-auth-c4c0d116", "name": "openshift-service-ca.crt", "reconcileID": "6c0eccd4-8aa5-42ed-a081-753b27c7f29b", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-c4c0d116"} 2026-07-08T19:37:56Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-c4c0d116"}, "namespace": "e2e-webhook-auth-c4c0d116", "name": "openshift-service-ca.crt", "reconcileID": "6c0eccd4-8aa5-42ed-a081-753b27c7f29b", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-c4c0d116"} 2026-07-08T19:37:56Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-c4c0d116"}, "namespace": "e2e-webhook-auth-c4c0d116", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "923bd1bf-02a5-4204-8a73-e72a29462ef3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-c4c0d116"} 2026-07-08T19:37:56Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-c4c0d116"}, "namespace": "e2e-webhook-auth-c4c0d116", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "923bd1bf-02a5-4204-8a73-e72a29462ef3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-c4c0d116"} 2026-07-08T19:38:04Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f3245bf3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3", "reconcileID": "12cda6d1-15ea-4ecb-8f00-20b501a8d962", "Gateway": "e2e-mt-f3245bf3", "namespace": "openshift-ingress"} 2026-07-08T19:38:04Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f3245bf3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3", "reconcileID": "12cda6d1-15ea-4ecb-8f00-20b501a8d962", "Gateway": "e2e-mt-f3245bf3", "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3-authn-ssl"} 2026-07-08T19:38:04Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f3245bf3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3", "reconcileID": "12cda6d1-15ea-4ecb-8f00-20b501a8d962", "Gateway": "e2e-mt-f3245bf3", "namespace": "openshift-ingress"} 2026-07-08T19:38:04Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f3245bf3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3", "reconcileID": "0eb7db74-4de0-492c-9c5f-2af77374a358", "Gateway": "e2e-mt-f3245bf3", "namespace": "openshift-ingress"} 2026-07-08T19:38:04Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f3245bf3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3", "reconcileID": "0eb7db74-4de0-492c-9c5f-2af77374a358", "Gateway": "e2e-mt-f3245bf3", "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3-authn-ssl"} 2026-07-08T19:38:04Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f3245bf3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3", "reconcileID": "0eb7db74-4de0-492c-9c5f-2af77374a358", "Gateway": "e2e-mt-f3245bf3", "namespace": "openshift-ingress"} 2026-07-08T19:38:04Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f3245bf3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3", "reconcileID": "11ff3c31-0e45-4027-9ca0-538d5d5c16a6", "Gateway": "e2e-mt-f3245bf3", "namespace": "openshift-ingress"} 2026-07-08T19:38:04Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f3245bf3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3", "reconcileID": "11ff3c31-0e45-4027-9ca0-538d5d5c16a6", "Gateway": "e2e-mt-f3245bf3", "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3-authn-ssl"} 2026-07-08T19:38:04Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f3245bf3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3", "reconcileID": "11ff3c31-0e45-4027-9ca0-538d5d5c16a6", "Gateway": "e2e-mt-f3245bf3", "namespace": "openshift-ingress"} 2026-07-08T19:38:04Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f3245bf3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3", "reconcileID": "a7d7fb4b-e9b4-4ac1-b484-8a336dba1a73", "Gateway": "e2e-mt-f3245bf3", "namespace": "openshift-ingress"} 2026-07-08T19:38:04Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f3245bf3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3", "reconcileID": "a7d7fb4b-e9b4-4ac1-b484-8a336dba1a73", "Gateway": "e2e-mt-f3245bf3", "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3-authn-ssl"} 2026-07-08T19:38:04Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f3245bf3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3", "reconcileID": "a7d7fb4b-e9b4-4ac1-b484-8a336dba1a73", "Gateway": "e2e-mt-f3245bf3", "namespace": "openshift-ingress"} 2026-07-08T19:38:04Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f3245bf3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3", "reconcileID": "6dca5c3b-dadf-4b68-9d25-957f640d3b64", "Gateway": "e2e-mt-f3245bf3", "namespace": "openshift-ingress"} 2026-07-08T19:38:04Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f3245bf3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3", "reconcileID": "6dca5c3b-dadf-4b68-9d25-957f640d3b64", "Gateway": "e2e-mt-f3245bf3", "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3-authn-ssl"} 2026-07-08T19:38:04Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f3245bf3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3", "reconcileID": "6dca5c3b-dadf-4b68-9d25-957f640d3b64", "Gateway": "e2e-mt-f3245bf3", "namespace": "openshift-ingress"} 2026-07-08T19:38:04Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f3245bf3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3", "reconcileID": "92c5d01e-fc74-47e4-942d-c1f99dee3131", "Gateway": "e2e-mt-f3245bf3", "namespace": "openshift-ingress"} 2026-07-08T19:38:04Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f3245bf3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3", "reconcileID": "92c5d01e-fc74-47e4-942d-c1f99dee3131", "Gateway": "e2e-mt-f3245bf3", "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3-authn-ssl"} 2026-07-08T19:38:04Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f3245bf3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3", "reconcileID": "92c5d01e-fc74-47e4-942d-c1f99dee3131", "Gateway": "e2e-mt-f3245bf3", "namespace": "openshift-ingress"} 2026-07-08T19:38:05Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f3245bf3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3", "reconcileID": "eab50252-d4ca-45ea-87d3-92b4d6014e7b", "Gateway": "e2e-mt-f3245bf3", "namespace": "openshift-ingress"} 2026-07-08T19:38:05Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f3245bf3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3", "reconcileID": "eab50252-d4ca-45ea-87d3-92b4d6014e7b", "Gateway": "e2e-mt-f3245bf3", "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3-authn-ssl"} 2026-07-08T19:38:05Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f3245bf3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3", "reconcileID": "eab50252-d4ca-45ea-87d3-92b4d6014e7b", "Gateway": "e2e-mt-f3245bf3", "namespace": "openshift-ingress"} 2026-07-08T19:41:11Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f3245bf3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f3245bf3", "reconcileID": "c674ec69-54e9-4f09-8b9a-2dd4763e0bba", "Gateway": "e2e-mt-f3245bf3", "namespace": "openshift-ingress"} 2026-07-08T19:41:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-f6a40ca3"}, "namespace": "ai-tenant-e2e-mt-f6a40ca3", "name": "odh-trusted-ca-bundle", "reconcileID": "40139e21-3291-434e-a0fc-12c00228f08c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-f6a40ca3"} 2026-07-08T19:41:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-f6a40ca3"}, "namespace": "ai-tenant-e2e-mt-f6a40ca3", "name": "openshift-service-ca.crt", "reconcileID": "121895d3-505a-49a4-8428-0471e3518bd5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-f6a40ca3"} 2026-07-08T19:41:13Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-f6a40ca3"}, "namespace": "ai-tenant-e2e-mt-f6a40ca3", "name": "openshift-service-ca.crt", "reconcileID": "11c9ea6e-c28b-41ac-b748-24f5780b1ca5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-f6a40ca3", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:41:13Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-f6a40ca3"}, "namespace": "ai-tenant-e2e-mt-f6a40ca3", "name": "openshift-service-ca.crt", "reconcileID": "11c9ea6e-c28b-41ac-b748-24f5780b1ca5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-f6a40ca3"} 2026-07-08T19:41:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-f6a40ca3"}, "namespace": "ai-tenant-e2e-mt-f6a40ca3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "99abf06b-7f24-400c-9385-0701b56a80bb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-f6a40ca3"} 2026-07-08T19:41:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-f6a40ca3"}, "namespace": "ai-tenant-e2e-mt-f6a40ca3", "name": "openshift-service-ca.crt", "reconcileID": "aeb81e65-3249-4207-bb00-4205971c52b0", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-f6a40ca3"} 2026-07-08T19:41:13Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-f6a40ca3"}, "namespace": "ai-tenant-e2e-mt-f6a40ca3", "name": "odh-trusted-ca-bundle", "reconcileID": "84058970-5f80-428a-8e5c-f717f3e76c9a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-f6a40ca3", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:41:13Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-f6a40ca3"}, "namespace": "ai-tenant-e2e-mt-f6a40ca3", "name": "odh-trusted-ca-bundle", "reconcileID": "84058970-5f80-428a-8e5c-f717f3e76c9a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-f6a40ca3"} 2026-07-08T19:41:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-f6a40ca3"}, "namespace": "ai-tenant-e2e-mt-f6a40ca3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a073f50f-31a2-4a13-88da-da969f3db60d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-f6a40ca3"} 2026-07-08T19:41:14Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-40c1e7f9"}, "namespace": "ai-tenant-e2e-mt-40c1e7f9", "name": "openshift-service-ca.crt", "reconcileID": "daf2493f-82cd-4a8f-864c-0720efb6c63a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-40c1e7f9"} 2026-07-08T19:41:14Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-40c1e7f9"}, "namespace": "ai-tenant-e2e-mt-40c1e7f9", "name": "odh-trusted-ca-bundle", "reconcileID": "5e46cfc3-69c5-477a-a71a-e0b50180b508", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-40c1e7f9"} 2026-07-08T19:41:14Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-40c1e7f9"}, "namespace": "ai-tenant-e2e-mt-40c1e7f9", "name": "openshift-service-ca.crt", "reconcileID": "916a5de6-342c-4e20-a50b-e2dab5362e20", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-40c1e7f9", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:41:14Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-40c1e7f9"}, "namespace": "ai-tenant-e2e-mt-40c1e7f9", "name": "openshift-service-ca.crt", "reconcileID": "916a5de6-342c-4e20-a50b-e2dab5362e20", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-40c1e7f9"} 2026-07-08T19:41:14Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-40c1e7f9"}, "namespace": "ai-tenant-e2e-mt-40c1e7f9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "69913202-8519-40c2-8c74-c2829c2e5737", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-40c1e7f9"} 2026-07-08T19:41:14Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-40c1e7f9"}, "namespace": "ai-tenant-e2e-mt-40c1e7f9", "name": "openshift-service-ca.crt", "reconcileID": "be7899fc-bb56-4f24-982b-3138d1734c61", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-40c1e7f9"} 2026-07-08T19:41:14Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-40c1e7f9"}, "namespace": "ai-tenant-e2e-mt-40c1e7f9", "name": "odh-trusted-ca-bundle", "reconcileID": "b94d89ed-cc16-4b6c-b53e-76384bb73633", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-40c1e7f9", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:41:14Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-40c1e7f9"}, "namespace": "ai-tenant-e2e-mt-40c1e7f9", "name": "odh-trusted-ca-bundle", "reconcileID": "b94d89ed-cc16-4b6c-b53e-76384bb73633", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-40c1e7f9"} 2026-07-08T19:41:14Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-40c1e7f9"}, "namespace": "ai-tenant-e2e-mt-40c1e7f9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dc0e2394-77ee-4ac9-8a46-9e36e39e0fd1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-40c1e7f9"} 2026-07-08T19:41:20Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-f6a40ca3"}, "namespace": "ai-tenant-e2e-mt-f6a40ca3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7f8fe593-e59b-41b7-af19-a1f4e810c363", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-f6a40ca3"} 2026-07-08T19:41:20Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-f6a40ca3"}, "namespace": "ai-tenant-e2e-mt-f6a40ca3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7f8fe593-e59b-41b7-af19-a1f4e810c363", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-f6a40ca3", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:41:20Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-f6a40ca3"}, "namespace": "ai-tenant-e2e-mt-f6a40ca3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7f8fe593-e59b-41b7-af19-a1f4e810c363", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-f6a40ca3 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T19:41:20Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-f6a40ca3"}, "namespace": "ai-tenant-e2e-mt-f6a40ca3", "name": "odh-trusted-ca-bundle", "reconcileID": "eb5ebc17-54fb-4a90-9f96-c3897c20d9e3", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-f6a40ca3"} 2026-07-08T19:41:20Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-f6a40ca3"}, "namespace": "ai-tenant-e2e-mt-f6a40ca3", "name": "odh-trusted-ca-bundle", "reconcileID": "eb5ebc17-54fb-4a90-9f96-c3897c20d9e3", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-f6a40ca3"} 2026-07-08T19:41:20Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-f6a40ca3"}, "namespace": "ai-tenant-e2e-mt-f6a40ca3", "name": "openshift-service-ca.crt", "reconcileID": "5ca83766-cd0c-4ca7-9fd7-f71ef851a442", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-f6a40ca3"} 2026-07-08T19:41:20Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-f6a40ca3"}, "namespace": "ai-tenant-e2e-mt-f6a40ca3", "name": "openshift-service-ca.crt", "reconcileID": "5ca83766-cd0c-4ca7-9fd7-f71ef851a442", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-f6a40ca3"} 2026-07-08T19:41:20Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-f6a40ca3"}, "namespace": "ai-tenant-e2e-mt-f6a40ca3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2c6d66fd-077e-47e1-b9b3-52293379052d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-f6a40ca3"} 2026-07-08T19:41:20Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-f6a40ca3"}, "namespace": "ai-tenant-e2e-mt-f6a40ca3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2c6d66fd-077e-47e1-b9b3-52293379052d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-f6a40ca3"} 2026-07-08T19:41:27Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-40c1e7f9"}, "namespace": "ai-tenant-e2e-mt-40c1e7f9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "93a18c64-e0f5-4518-abfa-e20c55e92c43", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-40c1e7f9"} 2026-07-08T19:41:27Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-40c1e7f9"}, "namespace": "ai-tenant-e2e-mt-40c1e7f9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "93a18c64-e0f5-4518-abfa-e20c55e92c43", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-40c1e7f9", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:41:27Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-40c1e7f9"}, "namespace": "ai-tenant-e2e-mt-40c1e7f9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "93a18c64-e0f5-4518-abfa-e20c55e92c43", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-40c1e7f9 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T19:41:27Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-40c1e7f9"}, "namespace": "ai-tenant-e2e-mt-40c1e7f9", "name": "odh-trusted-ca-bundle", "reconcileID": "977ae885-8851-46fa-906c-9100a42c9f8c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-40c1e7f9"} 2026-07-08T19:41:27Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-40c1e7f9"}, "namespace": "ai-tenant-e2e-mt-40c1e7f9", "name": "odh-trusted-ca-bundle", "reconcileID": "977ae885-8851-46fa-906c-9100a42c9f8c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-40c1e7f9"} 2026-07-08T19:41:27Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-40c1e7f9"}, "namespace": "ai-tenant-e2e-mt-40c1e7f9", "name": "openshift-service-ca.crt", "reconcileID": "04261ff8-22fc-496d-ae72-1ce7b80f2307", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-40c1e7f9"} 2026-07-08T19:41:27Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-40c1e7f9"}, "namespace": "ai-tenant-e2e-mt-40c1e7f9", "name": "openshift-service-ca.crt", "reconcileID": "04261ff8-22fc-496d-ae72-1ce7b80f2307", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-40c1e7f9"} 2026-07-08T19:41:27Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-40c1e7f9"}, "namespace": "ai-tenant-e2e-mt-40c1e7f9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "40a0b0ed-6720-403f-88cb-d8ca95e2c3c7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-40c1e7f9"} 2026-07-08T19:41:27Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-40c1e7f9"}, "namespace": "ai-tenant-e2e-mt-40c1e7f9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "40a0b0ed-6720-403f-88cb-d8ca95e2c3c7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-40c1e7f9"} 2026-07-08T19:41:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a55dec8a"}, "namespace": "ai-tenant-e2e-mt-a55dec8a", "name": "openshift-service-ca.crt", "reconcileID": "bdf4e215-eadd-451d-a9b0-26b09caccd37", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a55dec8a"} 2026-07-08T19:41:28Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a55dec8a"}, "namespace": "ai-tenant-e2e-mt-a55dec8a", "name": "openshift-service-ca.crt", "reconcileID": "e7368adc-444a-4b0f-a082-9ee7f2515e29", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a55dec8a", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:41:29Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a55dec8a"}, "namespace": "ai-tenant-e2e-mt-a55dec8a", "name": "openshift-service-ca.crt", "reconcileID": "e7368adc-444a-4b0f-a082-9ee7f2515e29", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a55dec8a"} 2026-07-08T19:41:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a55dec8a"}, "namespace": "ai-tenant-e2e-mt-a55dec8a", "name": "odh-trusted-ca-bundle", "reconcileID": "7c014034-6f61-49fa-9f25-8c790557a6b3", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a55dec8a"} 2026-07-08T19:41:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a55dec8a"}, "namespace": "ai-tenant-e2e-mt-a55dec8a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6f8d3675-a1ce-4da2-82d6-70f1838803dd", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a55dec8a"} 2026-07-08T19:41:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a55dec8a"}, "namespace": "ai-tenant-e2e-mt-a55dec8a", "name": "openshift-service-ca.crt", "reconcileID": "8c34954f-878d-4436-8066-92c33d690fc1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a55dec8a"} 2026-07-08T19:41:29Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a55dec8a"}, "namespace": "ai-tenant-e2e-mt-a55dec8a", "name": "odh-trusted-ca-bundle", "reconcileID": "98cde333-d3a8-412c-8528-eef0e9267496", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a55dec8a", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:41:29Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a55dec8a"}, "namespace": "ai-tenant-e2e-mt-a55dec8a", "name": "odh-trusted-ca-bundle", "reconcileID": "98cde333-d3a8-412c-8528-eef0e9267496", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a55dec8a"} 2026-07-08T19:41:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a55dec8a"}, "namespace": "ai-tenant-e2e-mt-a55dec8a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e44886c1-176e-416d-9cda-fb01bbef8d81", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a55dec8a"} 2026-07-08T19:41:54Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-08T19:41:54Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-08T19:41:55Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-08T19:41:55Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-08T19:42:06Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a55dec8a"}, "namespace": "ai-tenant-e2e-mt-a55dec8a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6dbfe8d5-d053-45ed-aaf0-447b410dc8b6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a55dec8a"} 2026-07-08T19:42:06Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a55dec8a"}, "namespace": "ai-tenant-e2e-mt-a55dec8a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6dbfe8d5-d053-45ed-aaf0-447b410dc8b6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a55dec8a", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T19:42:06Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a55dec8a"}, "namespace": "ai-tenant-e2e-mt-a55dec8a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6dbfe8d5-d053-45ed-aaf0-447b410dc8b6", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-a55dec8a because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T19:42:06Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a55dec8a"}, "namespace": "ai-tenant-e2e-mt-a55dec8a", "name": "odh-trusted-ca-bundle", "reconcileID": "b01730bb-ce16-41fb-81d0-f964dacc59b4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a55dec8a"} 2026-07-08T19:42:06Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a55dec8a"}, "namespace": "ai-tenant-e2e-mt-a55dec8a", "name": "odh-trusted-ca-bundle", "reconcileID": "b01730bb-ce16-41fb-81d0-f964dacc59b4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a55dec8a"} 2026-07-08T19:42:06Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a55dec8a"}, "namespace": "ai-tenant-e2e-mt-a55dec8a", "name": "openshift-service-ca.crt", "reconcileID": "1d13f2e1-f980-4770-945a-916112aea811", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a55dec8a"} 2026-07-08T19:42:06Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a55dec8a"}, "namespace": "ai-tenant-e2e-mt-a55dec8a", "name": "openshift-service-ca.crt", "reconcileID": "1d13f2e1-f980-4770-945a-916112aea811", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a55dec8a"} 2026-07-08T19:42:06Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a55dec8a"}, "namespace": "ai-tenant-e2e-mt-a55dec8a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d7ac6ccb-67ad-4769-80e7-5dbf8019d19a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a55dec8a"} 2026-07-08T19:42:06Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a55dec8a"}, "namespace": "ai-tenant-e2e-mt-a55dec8a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d7ac6ccb-67ad-4769-80e7-5dbf8019d19a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a55dec8a"} 2026-07-08T19:42:07Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "aba59ebc-64e2-4a69-8dc0-0cc9ae8dadab", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress"} 2026-07-08T19:42:07Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "aba59ebc-64e2-4a69-8dc0-0cc9ae8dadab", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5-authn-ssl"} 2026-07-08T19:42:07Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "aba59ebc-64e2-4a69-8dc0-0cc9ae8dadab", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress"} 2026-07-08T19:42:07Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "64a913d5-7ce2-4301-92fd-e1eed55d0669", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress"} 2026-07-08T19:42:07Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "64a913d5-7ce2-4301-92fd-e1eed55d0669", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5-authn-ssl"} 2026-07-08T19:42:07Z ERROR Reconciler error {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "64a913d5-7ce2-4301-92fd-e1eed55d0669", "error": "failed to create EnvoyFilter: could not CREATE EnvoyFilter openshift-ingress/e2e-inf-a-6511a5-authn-ssl: envoyfilters.networking.istio.io \"e2e-inf-a-6511a5-authn-ssl\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T19:42:07Z DEBUG events Failed to reconcile EnvoyFilter: failed to create EnvoyFilter: could not CREATE EnvoyFilter openshift-ingress/e2e-inf-a-6511a5-authn-ssl: envoyfilters.networking.istio.io "e2e-inf-a-6511a5-authn-ssl" already exists {"type": "Warning", "object": {"kind":"Gateway","namespace":"openshift-ingress","name":"e2e-inf-a-6511a5","uid":"32d138e0-e227-488d-a123-e84327a1d1b0","apiVersion":"gateway.networking.k8s.io/v1","resourceVersion":"51119"}, "reason": "ReconcileError"} 2026-07-08T19:42:07Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "43356a6a-8bd8-4fdc-8196-3f5760b85560", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress"} 2026-07-08T19:42:07Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "43356a6a-8bd8-4fdc-8196-3f5760b85560", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5-authn-ssl"} 2026-07-08T19:42:07Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "43356a6a-8bd8-4fdc-8196-3f5760b85560", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress"} 2026-07-08T19:42:07Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "3ffe9780-04ea-4e93-bbb4-436cf925f9b5", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress"} 2026-07-08T19:42:07Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "3ffe9780-04ea-4e93-bbb4-436cf925f9b5", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5-authn-ssl"} 2026-07-08T19:42:07Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "3ffe9780-04ea-4e93-bbb4-436cf925f9b5", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress"} 2026-07-08T19:42:07Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "4be4034d-7a70-4aea-aa1e-c82f0a1b0310", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress"} 2026-07-08T19:42:07Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "4be4034d-7a70-4aea-aa1e-c82f0a1b0310", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5-authn-ssl"} 2026-07-08T19:42:07Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "4be4034d-7a70-4aea-aa1e-c82f0a1b0310", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress"} 2026-07-08T19:42:08Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "c8dcc285-1159-403c-80f9-2506b362b0dc", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress"} 2026-07-08T19:42:08Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "c8dcc285-1159-403c-80f9-2506b362b0dc", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5-authn-ssl"} 2026-07-08T19:42:08Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "c8dcc285-1159-403c-80f9-2506b362b0dc", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress"} 2026-07-08T19:42:08Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "df03a396-8d09-41f7-b5c3-2b48e88b38c7", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress"} 2026-07-08T19:42:08Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "df03a396-8d09-41f7-b5c3-2b48e88b38c7", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5-authn-ssl"} 2026-07-08T19:42:08Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "df03a396-8d09-41f7-b5c3-2b48e88b38c7", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress"} 2026-07-08T19:42:09Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "aa5663cd-18fe-4933-af04-f381e27cd4f1", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress"} 2026-07-08T19:42:09Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "aa5663cd-18fe-4933-af04-f381e27cd4f1", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5-authn-ssl"} 2026-07-08T19:42:09Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "aa5663cd-18fe-4933-af04-f381e27cd4f1", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress"} 2026-07-08T19:45:15Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-6511a5","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-6511a5", "reconcileID": "a2e55a30-28af-4572-af1b-3cae2b7920ab", "Gateway": "e2e-inf-a-6511a5", "namespace": "openshift-ingress"}