2026-07-09T04:22:12Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "openshift-service-ca.crt", "reconcileID": "f7b3c7da-4995-4fc1-a9ee-8df1b007fad3", "ConfigMap": "openshift-service-ca.crt", "namespace": "premium-users-namespace", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:22:12Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "openshift-service-ca.crt", "reconcileID": "f7b3c7da-4995-4fc1-a9ee-8df1b007fad3", "ConfigMap": "openshift-service-ca.crt", "namespace": "premium-users-namespace"} 2026-07-09T04:22:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "df1f57c3-7895-466c-8e48-551dcdf3c2ea", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "premium-users-namespace"} 2026-07-09T04:22:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "openshift-service-ca.crt", "reconcileID": "e5f386eb-56b3-4416-8e7e-62b36af1366e", "ConfigMap": "openshift-service-ca.crt", "namespace": "premium-users-namespace"} 2026-07-09T04:22:12Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "odh-trusted-ca-bundle", "reconcileID": "25a518b6-b298-4e67-9428-dfbee2855a2b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "premium-users-namespace", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:22:12Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "odh-trusted-ca-bundle", "reconcileID": "25a518b6-b298-4e67-9428-dfbee2855a2b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "premium-users-namespace"} 2026-07-09T04:22:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ed9e80ae-7316-4ec4-8aed-6b844feea848", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "premium-users-namespace"} 2026-07-09T04:22:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "odh-trusted-ca-bundle", "reconcileID": "e6687fbf-3f1e-4026-9edd-932f60ff0bb7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "maas-admin"} 2026-07-09T04:22:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "openshift-service-ca.crt", "reconcileID": "66184db9-fd2d-42dd-a400-fdb857953923", "ConfigMap": "openshift-service-ca.crt", "namespace": "maas-admin"} 2026-07-09T04:22:13Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "openshift-service-ca.crt", "reconcileID": "4c35e847-e968-4d6f-bf8c-7d098c28db64", "ConfigMap": "openshift-service-ca.crt", "namespace": "maas-admin", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:22:13Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "openshift-service-ca.crt", "reconcileID": "4c35e847-e968-4d6f-bf8c-7d098c28db64", "ConfigMap": "openshift-service-ca.crt", "namespace": "maas-admin"} 2026-07-09T04:22:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cd283c9e-e3ac-4a94-b566-8b5f64e0e83a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "maas-admin"} 2026-07-09T04:22:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "openshift-service-ca.crt", "reconcileID": "512c9531-ea7b-4b65-beb5-1688a74d86b2", "ConfigMap": "openshift-service-ca.crt", "namespace": "maas-admin"} 2026-07-09T04:22:13Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "odh-trusted-ca-bundle", "reconcileID": "5516364a-b1ea-42a3-a0bb-3ab59a817ace", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "maas-admin", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:22:13Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "odh-trusted-ca-bundle", "reconcileID": "5516364a-b1ea-42a3-a0bb-3ab59a817ace", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "maas-admin"} 2026-07-09T04:22:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fa6ad682-cc0f-4217-90be-4ae84ec1ba38", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "maas-admin"} 2026-07-09T04:23:46Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "547b38a6-27b2-4ab5-9062-c035fb9add3d", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-07-09T04:23:46Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "547b38a6-27b2-4ab5-9062-c035fb9add3d", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-07-09T04:23:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "547b38a6-27b2-4ab5-9062-c035fb9add3d", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-07-09T04:23:52Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0d77f155-204b-4333-8cd9-bbb756c04372", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-07-09T04:23:52Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0d77f155-204b-4333-8cd9-bbb756c04372", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-07-09T04:23:52Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0d77f155-204b-4333-8cd9-bbb756c04372", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-07-09T04:24:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-87d226"}, "namespace": "e2e-webhook-test-87d226", "name": "openshift-service-ca.crt", "reconcileID": "672fc04d-e415-45c6-803a-bc868d658163", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-87d226"} 2026-07-09T04:24:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-87d226"}, "namespace": "e2e-webhook-test-87d226", "name": "odh-trusted-ca-bundle", "reconcileID": "b8938deb-0837-4913-8b0d-fac657e54892", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-87d226"} 2026-07-09T04:24:07Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-87d226"}, "namespace": "e2e-webhook-test-87d226", "name": "openshift-service-ca.crt", "reconcileID": "a684b15e-e31e-4492-9009-5e7a9a056202", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-87d226", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:24:07Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-87d226"}, "namespace": "e2e-webhook-test-87d226", "name": "openshift-service-ca.crt", "reconcileID": "a684b15e-e31e-4492-9009-5e7a9a056202", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-87d226"} 2026-07-09T04:24:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-87d226"}, "namespace": "e2e-webhook-test-87d226", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "aaec4dbd-190e-4a5b-a0bc-926f1541976b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-87d226"} 2026-07-09T04:24:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-87d226"}, "namespace": "e2e-webhook-test-87d226", "name": "openshift-service-ca.crt", "reconcileID": "b25a8c75-c6cd-410c-a96e-d0af1ac68d7c", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-87d226"} 2026-07-09T04:24:07Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-87d226"}, "namespace": "e2e-webhook-test-87d226", "name": "odh-trusted-ca-bundle", "reconcileID": "1353b87f-b24f-47d4-980a-4490daf9df8b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-87d226", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:24:07Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-87d226"}, "namespace": "e2e-webhook-test-87d226", "name": "odh-trusted-ca-bundle", "reconcileID": "1353b87f-b24f-47d4-980a-4490daf9df8b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-87d226"} 2026-07-09T04:24:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-87d226"}, "namespace": "e2e-webhook-test-87d226", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3cc3a0ed-a4cd-4d1c-93a5-3d9e8ef24a0a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-87d226"} 2026-07-09T04:24:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-87d226"}, "namespace": "e2e-webhook-test-87d226", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "731fa1d0-777c-4469-93d1-8d4fc245255c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-87d226"} 2026-07-09T04:24:13Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-87d226"}, "namespace": "e2e-webhook-test-87d226", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "731fa1d0-777c-4469-93d1-8d4fc245255c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-87d226", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:24:13Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-87d226"}, "namespace": "e2e-webhook-test-87d226", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "731fa1d0-777c-4469-93d1-8d4fc245255c", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-test-87d226 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T04:24:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-87d226"}, "namespace": "e2e-webhook-test-87d226", "name": "odh-trusted-ca-bundle", "reconcileID": "75b39636-de1f-490f-a7f3-f0c8b778f365", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-87d226"} 2026-07-09T04:24:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-87d226"}, "namespace": "e2e-webhook-test-87d226", "name": "odh-trusted-ca-bundle", "reconcileID": "75b39636-de1f-490f-a7f3-f0c8b778f365", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-87d226"} 2026-07-09T04:24:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-87d226"}, "namespace": "e2e-webhook-test-87d226", "name": "openshift-service-ca.crt", "reconcileID": "2bb7be7a-caed-4411-aa8a-6997c5c73465", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-87d226"} 2026-07-09T04:24:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-87d226"}, "namespace": "e2e-webhook-test-87d226", "name": "openshift-service-ca.crt", "reconcileID": "2bb7be7a-caed-4411-aa8a-6997c5c73465", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-87d226"} 2026-07-09T04:24:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-87d226"}, "namespace": "e2e-webhook-test-87d226", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "04deba52-28a3-454e-ae0b-4f83a43aceb6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-87d226"} 2026-07-09T04:24:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-87d226"}, "namespace": "e2e-webhook-test-87d226", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "04deba52-28a3-454e-ae0b-4f83a43aceb6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-87d226"} 2026-07-09T04:24:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-64f970"}, "namespace": "e2e-webhook-test-64f970", "name": "openshift-service-ca.crt", "reconcileID": "6a74f872-4fa9-4392-b514-2b6027b72bc1", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-64f970"} 2026-07-09T04:24:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-64f970"}, "namespace": "e2e-webhook-test-64f970", "name": "odh-trusted-ca-bundle", "reconcileID": "3aa32195-e46b-4b0f-977e-4c19971209d4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-64f970"} 2026-07-09T04:24:13Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-64f970"}, "namespace": "e2e-webhook-test-64f970", "name": "openshift-service-ca.crt", "reconcileID": "7d9bd734-d878-4956-b19f-b091d20e19e1", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-64f970", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:24:13Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-64f970"}, "namespace": "e2e-webhook-test-64f970", "name": "openshift-service-ca.crt", "reconcileID": "7d9bd734-d878-4956-b19f-b091d20e19e1", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-64f970"} 2026-07-09T04:24:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-64f970"}, "namespace": "e2e-webhook-test-64f970", "name": "openshift-service-ca.crt", "reconcileID": "c9f74230-25b6-4db9-aae6-f16bd543e60a", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-64f970"} 2026-07-09T04:24:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-64f970"}, "namespace": "e2e-webhook-test-64f970", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "56b5dad0-75cf-47d1-8729-ae1b27b7f212", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-64f970"} 2026-07-09T04:24:14Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-64f970"}, "namespace": "e2e-webhook-test-64f970", "name": "odh-trusted-ca-bundle", "reconcileID": "fb82947c-e0a2-4641-beac-93d15465ab69", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-64f970", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:24:14Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-64f970"}, "namespace": "e2e-webhook-test-64f970", "name": "odh-trusted-ca-bundle", "reconcileID": "fb82947c-e0a2-4641-beac-93d15465ab69", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-64f970"} 2026-07-09T04:24:14Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-64f970"}, "namespace": "e2e-webhook-test-64f970", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "384f1e7c-a37a-465f-b481-f37e7ecfd680", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-64f970"} 2026-07-09T04:24:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-64f970"}, "namespace": "e2e-webhook-test-64f970", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "85efa496-0ce3-4555-b7a9-c51d4e01b537", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-64f970"} 2026-07-09T04:24:19Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-64f970"}, "namespace": "e2e-webhook-test-64f970", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "85efa496-0ce3-4555-b7a9-c51d4e01b537", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-64f970", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:24:19Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-64f970"}, "namespace": "e2e-webhook-test-64f970", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "85efa496-0ce3-4555-b7a9-c51d4e01b537", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-test-64f970 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T04:24:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-64f970"}, "namespace": "e2e-webhook-test-64f970", "name": "odh-trusted-ca-bundle", "reconcileID": "ea116f64-a107-4c49-bed4-dde39a01f03a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-64f970"} 2026-07-09T04:24:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-test-64f970"}, "namespace": "e2e-webhook-test-64f970", "name": "odh-trusted-ca-bundle", "reconcileID": "ea116f64-a107-4c49-bed4-dde39a01f03a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-test-64f970"} 2026-07-09T04:24:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-64f970"}, "namespace": "e2e-webhook-test-64f970", "name": "openshift-service-ca.crt", "reconcileID": "5b3822f5-53b7-4299-84b6-20de8be278dc", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-64f970"} 2026-07-09T04:24:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-test-64f970"}, "namespace": "e2e-webhook-test-64f970", "name": "openshift-service-ca.crt", "reconcileID": "5b3822f5-53b7-4299-84b6-20de8be278dc", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-test-64f970"} 2026-07-09T04:24:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-64f970"}, "namespace": "e2e-webhook-test-64f970", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "58096db6-ecf3-4594-85ed-5812030b24bb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-64f970"} 2026-07-09T04:24:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-test-64f970"}, "namespace": "e2e-webhook-test-64f970", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "58096db6-ecf3-4594-85ed-5812030b24bb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-test-64f970"} 2026-07-09T04:25:09Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-09T04:25:09Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-09T04:25:11Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-09T04:25:11Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-09T04:33:20Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-09T04:33:20Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-09T04:33:21Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-09T04:33:21Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-09T04:35:04Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "b1accdd9-3277-466a-8208-2b2c806759db", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-07-09T04:35:04Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "b1accdd9-3277-466a-8208-2b2c806759db", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-07-09T04:35:04Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "b1accdd9-3277-466a-8208-2b2c806759db", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-07-09T04:35:12Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0b2fd5f5-aa14-4aeb-bcef-c07df8891549", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-07-09T04:35:12Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0b2fd5f5-aa14-4aeb-bcef-c07df8891549", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-07-09T04:35:12Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0b2fd5f5-aa14-4aeb-bcef-c07df8891549", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-07-09T04:35:20Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "11307610-dce5-4fd2-84d0-82e2fe853ef5", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-07-09T04:35:20Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "11307610-dce5-4fd2-84d0-82e2fe853ef5", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-07-09T04:35:20Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "11307610-dce5-4fd2-84d0-82e2fe853ef5", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-07-09T04:35:25Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0793626c-7c4c-4237-8752-484e32015f26", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-07-09T04:35:25Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0793626c-7c4c-4237-8752-484e32015f26", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-07-09T04:35:25Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0793626c-7c4c-4237-8752-484e32015f26", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-07-09T04:35:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-ait-wrong-887cc6d6"}, "namespace": "e2e-ait-wrong-887cc6d6", "name": "openshift-service-ca.crt", "reconcileID": "08fc932f-060d-41fe-903a-11543a51329d", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-ait-wrong-887cc6d6"} 2026-07-09T04:35:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-ait-wrong-887cc6d6"}, "namespace": "e2e-ait-wrong-887cc6d6", "name": "odh-trusted-ca-bundle", "reconcileID": "63f7789a-e13d-4a0f-b8ca-31a6121c7ffc", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-ait-wrong-887cc6d6"} 2026-07-09T04:35:28Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-ait-wrong-887cc6d6"}, "namespace": "e2e-ait-wrong-887cc6d6", "name": "openshift-service-ca.crt", "reconcileID": "d6a20637-6bc9-4428-a8bc-40fd7f63d23f", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-ait-wrong-887cc6d6", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:35:28Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-ait-wrong-887cc6d6"}, "namespace": "e2e-ait-wrong-887cc6d6", "name": "openshift-service-ca.crt", "reconcileID": "d6a20637-6bc9-4428-a8bc-40fd7f63d23f", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-ait-wrong-887cc6d6"} 2026-07-09T04:35:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-ait-wrong-887cc6d6"}, "namespace": "e2e-ait-wrong-887cc6d6", "name": "openshift-service-ca.crt", "reconcileID": "3f677901-0c59-4b6d-a5af-81e790c6f114", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-ait-wrong-887cc6d6"} 2026-07-09T04:35:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-ait-wrong-887cc6d6"}, "namespace": "e2e-ait-wrong-887cc6d6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "676eeead-b921-4ddf-b0c6-2224abe4f477", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-ait-wrong-887cc6d6"} 2026-07-09T04:35:28Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-ait-wrong-887cc6d6"}, "namespace": "e2e-ait-wrong-887cc6d6", "name": "odh-trusted-ca-bundle", "reconcileID": "ce443609-199c-489e-83af-1fa9f29fada9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-ait-wrong-887cc6d6", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:35:28Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-ait-wrong-887cc6d6"}, "namespace": "e2e-ait-wrong-887cc6d6", "name": "odh-trusted-ca-bundle", "reconcileID": "ce443609-199c-489e-83af-1fa9f29fada9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-ait-wrong-887cc6d6"} 2026-07-09T04:35:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-ait-wrong-887cc6d6"}, "namespace": "e2e-ait-wrong-887cc6d6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9e6b6a7d-4fe5-402b-8e36-1b2599ad4c38", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-ait-wrong-887cc6d6"} 2026-07-09T04:35:34Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-ait-wrong-887cc6d6"}, "namespace": "e2e-ait-wrong-887cc6d6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "21053f30-f6b0-4ae2-9092-c168a8256028", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-ait-wrong-887cc6d6"} 2026-07-09T04:35:34Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-ait-wrong-887cc6d6"}, "namespace": "e2e-ait-wrong-887cc6d6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "21053f30-f6b0-4ae2-9092-c168a8256028", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-ait-wrong-887cc6d6", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:35:34Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-ait-wrong-887cc6d6"}, "namespace": "e2e-ait-wrong-887cc6d6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "21053f30-f6b0-4ae2-9092-c168a8256028", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-ait-wrong-887cc6d6 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T04:35:34Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-ait-wrong-887cc6d6"}, "namespace": "e2e-ait-wrong-887cc6d6", "name": "odh-trusted-ca-bundle", "reconcileID": "f67e0931-82ca-41cb-8d93-cef561b885f4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-ait-wrong-887cc6d6"} 2026-07-09T04:35:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-ait-wrong-887cc6d6"}, "namespace": "e2e-ait-wrong-887cc6d6", "name": "odh-trusted-ca-bundle", "reconcileID": "f67e0931-82ca-41cb-8d93-cef561b885f4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-ait-wrong-887cc6d6"} 2026-07-09T04:35:34Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-ait-wrong-887cc6d6"}, "namespace": "e2e-ait-wrong-887cc6d6", "name": "openshift-service-ca.crt", "reconcileID": "16a84cd4-0102-462e-8193-b5a9ebe0e13d", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-ait-wrong-887cc6d6"} 2026-07-09T04:35:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-ait-wrong-887cc6d6"}, "namespace": "e2e-ait-wrong-887cc6d6", "name": "openshift-service-ca.crt", "reconcileID": "16a84cd4-0102-462e-8193-b5a9ebe0e13d", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-ait-wrong-887cc6d6"} 2026-07-09T04:35:34Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-ait-wrong-887cc6d6"}, "namespace": "e2e-ait-wrong-887cc6d6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "59c53ff6-d43c-4ede-a0ac-bb0ff4f9a10e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-ait-wrong-887cc6d6"} 2026-07-09T04:35:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-ait-wrong-887cc6d6"}, "namespace": "e2e-ait-wrong-887cc6d6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "59c53ff6-d43c-4ede-a0ac-bb0ff4f9a10e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-ait-wrong-887cc6d6"} 2026-07-09T04:35:35Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-51ad45e2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-51ad45e2", "reconcileID": "8aacc3fe-5750-46d0-8859-3f6e7d07b77e", "Gateway": "e2e-ait-51ad45e2", "namespace": "openshift-ingress"} 2026-07-09T04:35:35Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-51ad45e2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-51ad45e2", "reconcileID": "2d28dd05-0e6d-49ea-8bdd-80dc38266cfd", "Gateway": "e2e-ait-51ad45e2", "namespace": "openshift-ingress"} 2026-07-09T04:35:35Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-51ad45e2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-51ad45e2", "reconcileID": "51d70693-ed1d-4df3-b07f-d08adfe44554", "Gateway": "e2e-ait-51ad45e2", "namespace": "openshift-ingress"} 2026-07-09T04:35:35Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-51ad45e2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-51ad45e2", "reconcileID": "764088b7-811b-4b7f-aa4d-ee8f923b8a5d", "Gateway": "e2e-ait-51ad45e2", "namespace": "openshift-ingress"} 2026-07-09T04:35:35Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-51ad45e2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-51ad45e2", "reconcileID": "dfa2007e-eda7-4bad-88c2-dc18dae10d0b", "Gateway": "e2e-ait-51ad45e2", "namespace": "openshift-ingress"} 2026-07-09T04:35:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-51ad45e2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-51ad45e2", "reconcileID": "7352865e-4687-40b4-826f-95013cbc7a63", "Gateway": "e2e-ait-51ad45e2", "namespace": "openshift-ingress"} 2026-07-09T04:35:36Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-51ad45e2"}, "namespace": "ai-tenant-e2e-ait-51ad45e2", "name": "openshift-service-ca.crt", "reconcileID": "6a4fa6c6-84a1-43ff-8f85-70edde15b2a7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-51ad45e2"} 2026-07-09T04:35:36Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-51ad45e2"}, "namespace": "ai-tenant-e2e-ait-51ad45e2", "name": "odh-trusted-ca-bundle", "reconcileID": "5d9bdf5a-1b24-4c67-a38d-6e440394766e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-51ad45e2"} 2026-07-09T04:35:36Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-51ad45e2"}, "namespace": "ai-tenant-e2e-ait-51ad45e2", "name": "openshift-service-ca.crt", "reconcileID": "2153ab8c-eb6e-494d-a675-559d338d9497", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-51ad45e2", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:35:36Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-51ad45e2"}, "namespace": "ai-tenant-e2e-ait-51ad45e2", "name": "openshift-service-ca.crt", "reconcileID": "2153ab8c-eb6e-494d-a675-559d338d9497", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-51ad45e2"} 2026-07-09T04:35:36Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-51ad45e2"}, "namespace": "ai-tenant-e2e-ait-51ad45e2", "name": "openshift-service-ca.crt", "reconcileID": "5c420152-fc30-4f1c-a799-c796e2129ab3", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-51ad45e2"} 2026-07-09T04:35:36Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-51ad45e2"}, "namespace": "ai-tenant-e2e-ait-51ad45e2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9aabea03-36a4-476b-bcea-41ab5e5539fe", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-51ad45e2"} 2026-07-09T04:35:36Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-51ad45e2"}, "namespace": "ai-tenant-e2e-ait-51ad45e2", "name": "odh-trusted-ca-bundle", "reconcileID": "37866471-d637-478d-bbc5-6c5f2f6bc6b5", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-51ad45e2", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:35:36Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-51ad45e2"}, "namespace": "ai-tenant-e2e-ait-51ad45e2", "name": "odh-trusted-ca-bundle", "reconcileID": "37866471-d637-478d-bbc5-6c5f2f6bc6b5", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-51ad45e2"} 2026-07-09T04:35:36Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-51ad45e2"}, "namespace": "ai-tenant-e2e-ait-51ad45e2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f1dfe00a-af51-4ffc-a6f5-7d6481a1005a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-51ad45e2"} 2026-07-09T04:35:36Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "b33f0bfa-9882-42d6-883d-c40809fa5287", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-09T04:35:39Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-51ad45e2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-51ad45e2", "reconcileID": "28631d1d-ee52-484c-b8c2-337cc34bed20", "Gateway": "e2e-ait-51ad45e2", "namespace": "openshift-ingress"} 2026-07-09T04:35:45Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-51ad45e2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-51ad45e2", "reconcileID": "6836cb8e-8436-407b-91e6-c6b26eb4010a", "Gateway": "e2e-ait-51ad45e2", "namespace": "openshift-ingress"} 2026-07-09T04:36:47Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-51ad45e2"}, "namespace": "ai-tenant-e2e-ait-51ad45e2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "074769e7-9fe5-4647-935f-1e960a5f3c74", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-51ad45e2"} 2026-07-09T04:36:47Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-51ad45e2"}, "namespace": "ai-tenant-e2e-ait-51ad45e2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "074769e7-9fe5-4647-935f-1e960a5f3c74", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-51ad45e2", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:36:47Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-51ad45e2"}, "namespace": "ai-tenant-e2e-ait-51ad45e2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "074769e7-9fe5-4647-935f-1e960a5f3c74", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ait-51ad45e2 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T04:36:47Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-51ad45e2"}, "namespace": "ai-tenant-e2e-ait-51ad45e2", "name": "odh-trusted-ca-bundle", "reconcileID": "4c8d8ac5-e966-49b4-93d1-e11de8b4d933", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-51ad45e2"} 2026-07-09T04:36:47Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ait-51ad45e2"}, "namespace": "ai-tenant-e2e-ait-51ad45e2", "name": "odh-trusted-ca-bundle", "reconcileID": "4c8d8ac5-e966-49b4-93d1-e11de8b4d933", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ait-51ad45e2"} 2026-07-09T04:36:47Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-51ad45e2"}, "namespace": "ai-tenant-e2e-ait-51ad45e2", "name": "openshift-service-ca.crt", "reconcileID": "23ffaa53-3f1f-4d5f-8f61-85e8be49f5ab", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-51ad45e2"} 2026-07-09T04:36:47Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ait-51ad45e2"}, "namespace": "ai-tenant-e2e-ait-51ad45e2", "name": "openshift-service-ca.crt", "reconcileID": "23ffaa53-3f1f-4d5f-8f61-85e8be49f5ab", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ait-51ad45e2"} 2026-07-09T04:36:47Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-51ad45e2"}, "namespace": "ai-tenant-e2e-ait-51ad45e2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9211f8bf-ea75-45ab-8857-c87d805796e6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-51ad45e2"} 2026-07-09T04:36:47Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ait-51ad45e2"}, "namespace": "ai-tenant-e2e-ait-51ad45e2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9211f8bf-ea75-45ab-8857-c87d805796e6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ait-51ad45e2"} 2026-07-09T04:36:53Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-82cecee7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-82cecee7", "reconcileID": "b3e798e9-6b25-4671-a612-de20ff19f1f9", "Gateway": "e2e-ait-82cecee7", "namespace": "openshift-ingress"} 2026-07-09T04:36:53Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-82cecee7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-82cecee7", "reconcileID": "bec5041c-72e4-46d3-891c-b1c748900413", "Gateway": "e2e-ait-82cecee7", "namespace": "openshift-ingress"} 2026-07-09T04:36:53Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-82cecee7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-82cecee7", "reconcileID": "64dbfe18-293d-4a05-91c9-13cd950bcc48", "Gateway": "e2e-ait-82cecee7", "namespace": "openshift-ingress"} 2026-07-09T04:36:53Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-82cecee7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-82cecee7", "reconcileID": "02faea73-d3ba-4bd5-a2bd-d6d74cadb027", "Gateway": "e2e-ait-82cecee7", "namespace": "openshift-ingress"} 2026-07-09T04:36:53Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-82cecee7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-82cecee7", "reconcileID": "9974ae37-b2bb-4849-83b8-629adabedcc8", "Gateway": "e2e-ait-82cecee7", "namespace": "openshift-ingress"} 2026-07-09T04:36:54Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-82cecee7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-82cecee7", "reconcileID": "2696f6f5-0af9-4a68-a297-bddd1eb4f1fe", "Gateway": "e2e-ait-82cecee7", "namespace": "openshift-ingress"} 2026-07-09T04:36:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-82cecee7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-82cecee7", "reconcileID": "c80ba5cd-31e4-40c8-bc4f-3470ca785cd5", "Gateway": "e2e-ait-82cecee7", "namespace": "openshift-ingress"} 2026-07-09T04:38:47Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "1ef97a08-0607-4afd-97cd-c51e77341f9a", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-09T04:39:53Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-09T04:39:53Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-09T04:39:58Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-c5d9fea6","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-c5d9fea6", "reconcileID": "d6aee68a-d904-4c50-9830-9483ee1ab7ae", "Gateway": "e2e-derive-c5d9fea6", "namespace": "openshift-ingress"} 2026-07-09T04:39:58Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-c5d9fea6","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-c5d9fea6", "reconcileID": "1b054bd0-08e7-4aeb-8885-9b257417bf81", "Gateway": "e2e-derive-c5d9fea6", "namespace": "openshift-ingress"} 2026-07-09T04:39:58Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-c5d9fea6","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-c5d9fea6", "reconcileID": "06ef148c-db59-4c55-9919-c5d1ff216ca6", "Gateway": "e2e-derive-c5d9fea6", "namespace": "openshift-ingress"} 2026-07-09T04:39:58Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-c5d9fea6","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-c5d9fea6", "reconcileID": "552101e2-6165-428a-9aca-89ee0270d3c6", "Gateway": "e2e-derive-c5d9fea6", "namespace": "openshift-ingress"} 2026-07-09T04:39:58Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-c5d9fea6","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-c5d9fea6", "reconcileID": "91040334-e5b5-4a63-b55b-33487e262f24", "Gateway": "e2e-derive-c5d9fea6", "namespace": "openshift-ingress"} 2026-07-09T04:39:59Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-c5d9fea6","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-c5d9fea6", "reconcileID": "c7289606-8d8c-4186-9396-972c932cf09b", "Gateway": "e2e-derive-c5d9fea6", "namespace": "openshift-ingress"} 2026-07-09T04:40:01Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-09T04:40:01Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-09T04:40:23Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-derive-c5d9fea6","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-derive-c5d9fea6", "reconcileID": "944f0309-2a3e-4ba2-a177-66e488bbd261", "Gateway": "e2e-derive-c5d9fea6", "namespace": "openshift-ingress"} 2026-07-09T04:42:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-7fc7db67"}, "namespace": "ai-tenant-e2e-mt-7fc7db67", "name": "openshift-service-ca.crt", "reconcileID": "8e4ef3b3-c3d4-467d-b3b4-726faa69fa97", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-7fc7db67"} 2026-07-09T04:42:02Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-7fc7db67"}, "namespace": "ai-tenant-e2e-mt-7fc7db67", "name": "openshift-service-ca.crt", "reconcileID": "4abcaa44-b649-4404-8758-8cfcee788952", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-7fc7db67", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:42:02Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-7fc7db67"}, "namespace": "ai-tenant-e2e-mt-7fc7db67", "name": "openshift-service-ca.crt", "reconcileID": "4abcaa44-b649-4404-8758-8cfcee788952", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-7fc7db67"} 2026-07-09T04:42:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-7fc7db67"}, "namespace": "ai-tenant-e2e-mt-7fc7db67", "name": "odh-trusted-ca-bundle", "reconcileID": "3495f64e-4269-4f6b-bf2d-458da50fd157", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-7fc7db67"} 2026-07-09T04:42:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-7fc7db67"}, "namespace": "ai-tenant-e2e-mt-7fc7db67", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dcdbf983-ee60-4f8d-b5b7-70df7e42000c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-7fc7db67"} 2026-07-09T04:42:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-7fc7db67"}, "namespace": "ai-tenant-e2e-mt-7fc7db67", "name": "openshift-service-ca.crt", "reconcileID": "961bfd29-761c-40d5-81db-8bba5a34adbd", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-7fc7db67"} 2026-07-09T04:42:02Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-7fc7db67"}, "namespace": "ai-tenant-e2e-mt-7fc7db67", "name": "odh-trusted-ca-bundle", "reconcileID": "3b9fb364-801c-4712-bb13-22343d35d336", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-7fc7db67", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:42:02Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-7fc7db67"}, "namespace": "ai-tenant-e2e-mt-7fc7db67", "name": "odh-trusted-ca-bundle", "reconcileID": "3b9fb364-801c-4712-bb13-22343d35d336", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-7fc7db67"} 2026-07-09T04:42:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-7fc7db67"}, "namespace": "ai-tenant-e2e-mt-7fc7db67", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5cb97cdb-22f8-407a-9199-af2228b833ed", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-7fc7db67"} 2026-07-09T04:42:09Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-7fc7db67"}, "namespace": "ai-tenant-e2e-mt-7fc7db67", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "55c99acd-3ab8-4004-b3b1-2d316e33159b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-7fc7db67"} 2026-07-09T04:42:09Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-7fc7db67"}, "namespace": "ai-tenant-e2e-mt-7fc7db67", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "55c99acd-3ab8-4004-b3b1-2d316e33159b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-7fc7db67", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:42:09Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-7fc7db67"}, "namespace": "ai-tenant-e2e-mt-7fc7db67", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "55c99acd-3ab8-4004-b3b1-2d316e33159b", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-7fc7db67 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T04:42:09Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-7fc7db67"}, "namespace": "ai-tenant-e2e-mt-7fc7db67", "name": "odh-trusted-ca-bundle", "reconcileID": "1c2c5c09-07f3-4cd9-beff-87d3572cc14f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-7fc7db67"} 2026-07-09T04:42:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-7fc7db67"}, "namespace": "ai-tenant-e2e-mt-7fc7db67", "name": "odh-trusted-ca-bundle", "reconcileID": "1c2c5c09-07f3-4cd9-beff-87d3572cc14f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-7fc7db67"} 2026-07-09T04:42:09Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-7fc7db67"}, "namespace": "ai-tenant-e2e-mt-7fc7db67", "name": "openshift-service-ca.crt", "reconcileID": "f5a171b7-e872-4765-8a74-21b552d2cfa9", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-7fc7db67"} 2026-07-09T04:42:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-7fc7db67"}, "namespace": "ai-tenant-e2e-mt-7fc7db67", "name": "openshift-service-ca.crt", "reconcileID": "f5a171b7-e872-4765-8a74-21b552d2cfa9", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-7fc7db67"} 2026-07-09T04:42:09Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-7fc7db67"}, "namespace": "ai-tenant-e2e-mt-7fc7db67", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6c274864-0192-4fc2-b42f-3fa4bdbce473", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-7fc7db67"} 2026-07-09T04:42:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-7fc7db67"}, "namespace": "ai-tenant-e2e-mt-7fc7db67", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6c274864-0192-4fc2-b42f-3fa4bdbce473", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-7fc7db67"} 2026-07-09T04:42:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-5958cc43"}, "namespace": "ai-tenant-e2e-mt-5958cc43", "name": "openshift-service-ca.crt", "reconcileID": "9918dce8-1e42-4cd5-bc45-2575b5f76ee7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-5958cc43"} 2026-07-09T04:42:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-5958cc43"}, "namespace": "ai-tenant-e2e-mt-5958cc43", "name": "odh-trusted-ca-bundle", "reconcileID": "86bb1614-4c0e-4536-aaa5-0ad88687ad62", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-5958cc43"} 2026-07-09T04:42:09Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-5958cc43"}, "namespace": "ai-tenant-e2e-mt-5958cc43", "name": "openshift-service-ca.crt", "reconcileID": "529d2d09-8274-43de-bb6c-9cf32921bc85", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-5958cc43", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:42:09Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-5958cc43"}, "namespace": "ai-tenant-e2e-mt-5958cc43", "name": "openshift-service-ca.crt", "reconcileID": "529d2d09-8274-43de-bb6c-9cf32921bc85", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-5958cc43"} 2026-07-09T04:42:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-5958cc43"}, "namespace": "ai-tenant-e2e-mt-5958cc43", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d5242eaa-11f3-4b26-920c-554cbba4f859", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-5958cc43"} 2026-07-09T04:42:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-5958cc43"}, "namespace": "ai-tenant-e2e-mt-5958cc43", "name": "openshift-service-ca.crt", "reconcileID": "18c6e3af-332f-4ce7-9619-8d58049c8b9a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-5958cc43"} 2026-07-09T04:42:09Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-5958cc43"}, "namespace": "ai-tenant-e2e-mt-5958cc43", "name": "odh-trusted-ca-bundle", "reconcileID": "3013e869-302f-4d52-bc95-6c0fa46374b0", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-5958cc43", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:42:09Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-5958cc43"}, "namespace": "ai-tenant-e2e-mt-5958cc43", "name": "odh-trusted-ca-bundle", "reconcileID": "3013e869-302f-4d52-bc95-6c0fa46374b0", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-5958cc43"} 2026-07-09T04:42:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-5958cc43"}, "namespace": "ai-tenant-e2e-mt-5958cc43", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c4607a94-75bc-4020-adf3-dab6f3c03e45", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-5958cc43"} 2026-07-09T04:42:41Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-5958cc43"}, "namespace": "ai-tenant-e2e-mt-5958cc43", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7142e524-5b15-42ca-8f0f-c7bb20354cce", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-5958cc43"} 2026-07-09T04:42:41Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-5958cc43"}, "namespace": "ai-tenant-e2e-mt-5958cc43", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7142e524-5b15-42ca-8f0f-c7bb20354cce", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-5958cc43", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:42:41Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-5958cc43"}, "namespace": "ai-tenant-e2e-mt-5958cc43", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7142e524-5b15-42ca-8f0f-c7bb20354cce", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-5958cc43 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T04:42:41Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-5958cc43"}, "namespace": "ai-tenant-e2e-mt-5958cc43", "name": "odh-trusted-ca-bundle", "reconcileID": "083e5fed-e03a-41e9-a51c-6fb289236b60", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-5958cc43"} 2026-07-09T04:42:41Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-5958cc43"}, "namespace": "ai-tenant-e2e-mt-5958cc43", "name": "odh-trusted-ca-bundle", "reconcileID": "083e5fed-e03a-41e9-a51c-6fb289236b60", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-5958cc43"} 2026-07-09T04:42:41Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-5958cc43"}, "namespace": "ai-tenant-e2e-mt-5958cc43", "name": "openshift-service-ca.crt", "reconcileID": "6252dd1c-f0c7-4e67-b1dd-04d6feef9505", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-5958cc43"} 2026-07-09T04:42:41Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-5958cc43"}, "namespace": "ai-tenant-e2e-mt-5958cc43", "name": "openshift-service-ca.crt", "reconcileID": "6252dd1c-f0c7-4e67-b1dd-04d6feef9505", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-5958cc43"} 2026-07-09T04:42:41Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-5958cc43"}, "namespace": "ai-tenant-e2e-mt-5958cc43", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "05c17113-1050-40a8-8dcb-e9ffd94c8237", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-5958cc43"} 2026-07-09T04:42:41Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-5958cc43"}, "namespace": "ai-tenant-e2e-mt-5958cc43", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "05c17113-1050-40a8-8dcb-e9ffd94c8237", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-5958cc43"} 2026-07-09T04:42:42Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-2828a73d"}, "namespace": "e2e-unlabeled-2828a73d", "name": "openshift-service-ca.crt", "reconcileID": "a88e41c8-ffdb-41cc-96f2-d9ffefec4d78", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-2828a73d"} 2026-07-09T04:42:42Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-2828a73d"}, "namespace": "e2e-unlabeled-2828a73d", "name": "odh-trusted-ca-bundle", "reconcileID": "58606907-a136-4a50-b796-a042fc53e919", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-2828a73d"} 2026-07-09T04:42:42Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-2828a73d"}, "namespace": "e2e-unlabeled-2828a73d", "name": "openshift-service-ca.crt", "reconcileID": "f7fe9d66-089e-4f90-9912-3b496e65989d", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-2828a73d", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:42:42Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-2828a73d"}, "namespace": "e2e-unlabeled-2828a73d", "name": "openshift-service-ca.crt", "reconcileID": "f7fe9d66-089e-4f90-9912-3b496e65989d", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-2828a73d"} 2026-07-09T04:42:42Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-2828a73d"}, "namespace": "e2e-unlabeled-2828a73d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "20bc2c19-ef54-43c4-98a1-623847bc51d7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-2828a73d"} 2026-07-09T04:42:42Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-2828a73d"}, "namespace": "e2e-unlabeled-2828a73d", "name": "openshift-service-ca.crt", "reconcileID": "ca2d6294-262a-4d8b-908a-8df9c9b77585", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-2828a73d"} 2026-07-09T04:42:42Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-2828a73d"}, "namespace": "e2e-unlabeled-2828a73d", "name": "odh-trusted-ca-bundle", "reconcileID": "8a27e03d-0966-4f92-bb9f-69dad5a8f749", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-2828a73d", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:42:42Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-2828a73d"}, "namespace": "e2e-unlabeled-2828a73d", "name": "odh-trusted-ca-bundle", "reconcileID": "8a27e03d-0966-4f92-bb9f-69dad5a8f749", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-2828a73d"} 2026-07-09T04:42:42Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-2828a73d"}, "namespace": "e2e-unlabeled-2828a73d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3f17454b-7126-492e-ba37-abce6db24d43", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-2828a73d"} 2026-07-09T04:43:03Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-2828a73d"}, "namespace": "e2e-unlabeled-2828a73d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6f876d09-f5fd-4bae-85e5-7aa6594a6d50", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-2828a73d"} 2026-07-09T04:43:03Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-2828a73d"}, "namespace": "e2e-unlabeled-2828a73d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6f876d09-f5fd-4bae-85e5-7aa6594a6d50", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-2828a73d", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:43:03Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-2828a73d"}, "namespace": "e2e-unlabeled-2828a73d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6f876d09-f5fd-4bae-85e5-7aa6594a6d50", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-unlabeled-2828a73d because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T04:43:03Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-2828a73d"}, "namespace": "e2e-unlabeled-2828a73d", "name": "odh-trusted-ca-bundle", "reconcileID": "5ba50eaf-3c48-4a96-a7f7-3bcbbfd80b6e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-2828a73d"} 2026-07-09T04:43:03Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unlabeled-2828a73d"}, "namespace": "e2e-unlabeled-2828a73d", "name": "odh-trusted-ca-bundle", "reconcileID": "5ba50eaf-3c48-4a96-a7f7-3bcbbfd80b6e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unlabeled-2828a73d"} 2026-07-09T04:43:03Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-2828a73d"}, "namespace": "e2e-unlabeled-2828a73d", "name": "openshift-service-ca.crt", "reconcileID": "56d25a58-0d09-4c17-9a1d-b4813ddb5317", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-2828a73d"} 2026-07-09T04:43:03Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unlabeled-2828a73d"}, "namespace": "e2e-unlabeled-2828a73d", "name": "openshift-service-ca.crt", "reconcileID": "56d25a58-0d09-4c17-9a1d-b4813ddb5317", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unlabeled-2828a73d"} 2026-07-09T04:43:03Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-2828a73d"}, "namespace": "e2e-unlabeled-2828a73d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "06a9cc80-7a8c-4efa-b99a-723659fb01e9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-2828a73d"} 2026-07-09T04:43:03Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unlabeled-2828a73d"}, "namespace": "e2e-unlabeled-2828a73d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "06a9cc80-7a8c-4efa-b99a-723659fb01e9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unlabeled-2828a73d"} 2026-07-09T04:43:03Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-bdab048d"}, "namespace": "ai-tenant-e2e-mt-bdab048d", "name": "openshift-service-ca.crt", "reconcileID": "84e86d34-b2f2-4de2-a4d5-bb9a7219b5c0", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-bdab048d"} 2026-07-09T04:43:03Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-bdab048d"}, "namespace": "ai-tenant-e2e-mt-bdab048d", "name": "openshift-service-ca.crt", "reconcileID": "eded4318-a3d1-4c42-829b-cbc5b26f9998", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-bdab048d", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:43:03Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-bdab048d"}, "namespace": "ai-tenant-e2e-mt-bdab048d", "name": "openshift-service-ca.crt", "reconcileID": "eded4318-a3d1-4c42-829b-cbc5b26f9998", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-bdab048d"} 2026-07-09T04:43:03Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-bdab048d"}, "namespace": "ai-tenant-e2e-mt-bdab048d", "name": "odh-trusted-ca-bundle", "reconcileID": "2baa9b6d-ff74-4f21-bb0a-a3243aeeeaa9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-bdab048d"} 2026-07-09T04:43:03Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-bdab048d"}, "namespace": "ai-tenant-e2e-mt-bdab048d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0f2ba9e8-144e-445a-9946-9105a70e9a31", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-bdab048d"} 2026-07-09T04:43:03Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-bdab048d"}, "namespace": "ai-tenant-e2e-mt-bdab048d", "name": "openshift-service-ca.crt", "reconcileID": "9c5ece6b-a764-4c32-9f3a-43f4576fb038", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-bdab048d"} 2026-07-09T04:43:03Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-bdab048d"}, "namespace": "ai-tenant-e2e-mt-bdab048d", "name": "odh-trusted-ca-bundle", "reconcileID": "e3cd2061-68c6-46bf-b2a9-b9bcd020b546", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-bdab048d", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:43:03Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-bdab048d"}, "namespace": "ai-tenant-e2e-mt-bdab048d", "name": "odh-trusted-ca-bundle", "reconcileID": "e3cd2061-68c6-46bf-b2a9-b9bcd020b546", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-bdab048d"} 2026-07-09T04:43:03Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-bdab048d"}, "namespace": "ai-tenant-e2e-mt-bdab048d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ef812a51-567f-4b13-9f3f-68dc053a1b5a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-bdab048d"} 2026-07-09T04:43:20Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-bdab048d"}, "namespace": "ai-tenant-e2e-mt-bdab048d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8e69da07-e42a-4e68-afa5-38ea30cfbaff", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-bdab048d"} 2026-07-09T04:43:20Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-bdab048d"}, "namespace": "ai-tenant-e2e-mt-bdab048d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8e69da07-e42a-4e68-afa5-38ea30cfbaff", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-bdab048d", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:43:20Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-bdab048d"}, "namespace": "ai-tenant-e2e-mt-bdab048d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8e69da07-e42a-4e68-afa5-38ea30cfbaff", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-bdab048d because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T04:43:20Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-bdab048d"}, "namespace": "ai-tenant-e2e-mt-bdab048d", "name": "odh-trusted-ca-bundle", "reconcileID": "89e5f40a-ee97-40a1-8f1b-f483b7644e78", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-bdab048d"} 2026-07-09T04:43:20Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-bdab048d"}, "namespace": "ai-tenant-e2e-mt-bdab048d", "name": "odh-trusted-ca-bundle", "reconcileID": "89e5f40a-ee97-40a1-8f1b-f483b7644e78", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-bdab048d"} 2026-07-09T04:43:20Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-bdab048d"}, "namespace": "ai-tenant-e2e-mt-bdab048d", "name": "openshift-service-ca.crt", "reconcileID": "b1b2d7c5-b880-4e53-b6d3-a7aa673708e1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-bdab048d"} 2026-07-09T04:43:20Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-bdab048d"}, "namespace": "ai-tenant-e2e-mt-bdab048d", "name": "openshift-service-ca.crt", "reconcileID": "b1b2d7c5-b880-4e53-b6d3-a7aa673708e1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-bdab048d"} 2026-07-09T04:43:20Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-bdab048d"}, "namespace": "ai-tenant-e2e-mt-bdab048d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fe754421-419c-47a6-9ad9-8af902099242", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-bdab048d"} 2026-07-09T04:43:20Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-bdab048d"}, "namespace": "ai-tenant-e2e-mt-bdab048d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fe754421-419c-47a6-9ad9-8af902099242", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-bdab048d"} 2026-07-09T04:43:20Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-ee9d2a43"}, "namespace": "ai-tenant-e2e-mt-ee9d2a43", "name": "openshift-service-ca.crt", "reconcileID": "b3a643b5-8409-43d6-afba-6869d9c7b53f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-ee9d2a43"} 2026-07-09T04:43:20Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-ee9d2a43"}, "namespace": "ai-tenant-e2e-mt-ee9d2a43", "name": "odh-trusted-ca-bundle", "reconcileID": "01b1dd27-6603-4421-9360-6ca92ff0304d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-ee9d2a43"} 2026-07-09T04:43:20Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-ee9d2a43"}, "namespace": "ai-tenant-e2e-mt-ee9d2a43", "name": "openshift-service-ca.crt", "reconcileID": "9292f43f-cfe2-4449-93f5-30b00922b75a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-ee9d2a43", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:43:20Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-ee9d2a43"}, "namespace": "ai-tenant-e2e-mt-ee9d2a43", "name": "openshift-service-ca.crt", "reconcileID": "9292f43f-cfe2-4449-93f5-30b00922b75a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-ee9d2a43"} 2026-07-09T04:43:20Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-ee9d2a43"}, "namespace": "ai-tenant-e2e-mt-ee9d2a43", "name": "openshift-service-ca.crt", "reconcileID": "e122df16-0ae3-4191-9678-a67b678d9b5e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-ee9d2a43"} 2026-07-09T04:43:20Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-ee9d2a43"}, "namespace": "ai-tenant-e2e-mt-ee9d2a43", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7c39263b-4a99-4a37-818f-3d9ac307f9c8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-ee9d2a43"} 2026-07-09T04:43:20Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-ee9d2a43"}, "namespace": "ai-tenant-e2e-mt-ee9d2a43", "name": "odh-trusted-ca-bundle", "reconcileID": "934d500f-4cbb-444b-aead-2714ccb73d50", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-ee9d2a43", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:43:20Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-ee9d2a43"}, "namespace": "ai-tenant-e2e-mt-ee9d2a43", "name": "odh-trusted-ca-bundle", "reconcileID": "934d500f-4cbb-444b-aead-2714ccb73d50", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-ee9d2a43"} 2026-07-09T04:43:20Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-ee9d2a43"}, "namespace": "ai-tenant-e2e-mt-ee9d2a43", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0b215c14-ac9a-42f9-bd8b-b5763424d2a8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-ee9d2a43"} 2026-07-09T04:43:21Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-21b19766"}, "namespace": "ai-tenant-e2e-mt-21b19766", "name": "openshift-service-ca.crt", "reconcileID": "3dd42b75-49a6-4249-b9cd-f5235efe2f9e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-21b19766"} 2026-07-09T04:43:21Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-21b19766"}, "namespace": "ai-tenant-e2e-mt-21b19766", "name": "odh-trusted-ca-bundle", "reconcileID": "4cb9e02c-f033-491a-9898-0980c7b32bcc", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-21b19766"} 2026-07-09T04:43:21Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-21b19766"}, "namespace": "ai-tenant-e2e-mt-21b19766", "name": "openshift-service-ca.crt", "reconcileID": "80b70749-37cf-40ea-9340-ff89d1a0d1a8", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-21b19766", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:43:21Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-21b19766"}, "namespace": "ai-tenant-e2e-mt-21b19766", "name": "openshift-service-ca.crt", "reconcileID": "80b70749-37cf-40ea-9340-ff89d1a0d1a8", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-21b19766"} 2026-07-09T04:43:21Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-21b19766"}, "namespace": "ai-tenant-e2e-mt-21b19766", "name": "openshift-service-ca.crt", "reconcileID": "317302bd-9804-485d-b737-955f28d98b2d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-21b19766"} 2026-07-09T04:43:21Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-21b19766"}, "namespace": "ai-tenant-e2e-mt-21b19766", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "25d33966-fea1-4556-abb3-9d5a42959d74", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-21b19766"} 2026-07-09T04:43:21Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-21b19766"}, "namespace": "ai-tenant-e2e-mt-21b19766", "name": "odh-trusted-ca-bundle", "reconcileID": "e75512c4-674e-4959-9ce3-6e1568548c4d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-21b19766", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:43:21Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-21b19766"}, "namespace": "ai-tenant-e2e-mt-21b19766", "name": "odh-trusted-ca-bundle", "reconcileID": "e75512c4-674e-4959-9ce3-6e1568548c4d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-21b19766"} 2026-07-09T04:43:21Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-21b19766"}, "namespace": "ai-tenant-e2e-mt-21b19766", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1a62f956-daba-4aa3-bbc5-e8862761ce8f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-21b19766"} 2026-07-09T04:43:28Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-ee9d2a43"}, "namespace": "ai-tenant-e2e-mt-ee9d2a43", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "59f66d6d-adc6-49c6-be69-18d1b9f7c798", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-ee9d2a43"} 2026-07-09T04:43:28Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-ee9d2a43"}, "namespace": "ai-tenant-e2e-mt-ee9d2a43", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "59f66d6d-adc6-49c6-be69-18d1b9f7c798", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-ee9d2a43", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:43:28Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-ee9d2a43"}, "namespace": "ai-tenant-e2e-mt-ee9d2a43", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "59f66d6d-adc6-49c6-be69-18d1b9f7c798", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-ee9d2a43 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T04:43:28Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-ee9d2a43"}, "namespace": "ai-tenant-e2e-mt-ee9d2a43", "name": "odh-trusted-ca-bundle", "reconcileID": "e43ba321-19b5-41ce-9b18-f1d704109dcd", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-ee9d2a43"} 2026-07-09T04:43:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-ee9d2a43"}, "namespace": "ai-tenant-e2e-mt-ee9d2a43", "name": "odh-trusted-ca-bundle", "reconcileID": "e43ba321-19b5-41ce-9b18-f1d704109dcd", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-ee9d2a43"} 2026-07-09T04:43:28Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-ee9d2a43"}, "namespace": "ai-tenant-e2e-mt-ee9d2a43", "name": "openshift-service-ca.crt", "reconcileID": "6e8c31b5-0b63-4570-b2ad-326db3397d16", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-ee9d2a43"} 2026-07-09T04:43:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-ee9d2a43"}, "namespace": "ai-tenant-e2e-mt-ee9d2a43", "name": "openshift-service-ca.crt", "reconcileID": "6e8c31b5-0b63-4570-b2ad-326db3397d16", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-ee9d2a43"} 2026-07-09T04:43:28Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-ee9d2a43"}, "namespace": "ai-tenant-e2e-mt-ee9d2a43", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "574ff8c6-1892-4a91-9c9f-c7b67bccc901", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-ee9d2a43"} 2026-07-09T04:43:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-ee9d2a43"}, "namespace": "ai-tenant-e2e-mt-ee9d2a43", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "574ff8c6-1892-4a91-9c9f-c7b67bccc901", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-ee9d2a43"} 2026-07-09T04:43:35Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-21b19766"}, "namespace": "ai-tenant-e2e-mt-21b19766", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ccd4e5bb-f99f-43d0-94ff-2d151f48971a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-21b19766"} 2026-07-09T04:43:35Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-21b19766"}, "namespace": "ai-tenant-e2e-mt-21b19766", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ccd4e5bb-f99f-43d0-94ff-2d151f48971a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-21b19766", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:43:35Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-21b19766"}, "namespace": "ai-tenant-e2e-mt-21b19766", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ccd4e5bb-f99f-43d0-94ff-2d151f48971a", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-21b19766 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T04:43:35Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-21b19766"}, "namespace": "ai-tenant-e2e-mt-21b19766", "name": "odh-trusted-ca-bundle", "reconcileID": "12cc37ff-52fd-4faf-b3cf-79e944f926a3", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-21b19766"} 2026-07-09T04:43:35Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-21b19766"}, "namespace": "ai-tenant-e2e-mt-21b19766", "name": "odh-trusted-ca-bundle", "reconcileID": "12cc37ff-52fd-4faf-b3cf-79e944f926a3", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-21b19766"} 2026-07-09T04:43:35Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-21b19766"}, "namespace": "ai-tenant-e2e-mt-21b19766", "name": "openshift-service-ca.crt", "reconcileID": "ca9d05b9-9c59-4647-abde-a4f76cf99a48", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-21b19766"} 2026-07-09T04:43:35Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-21b19766"}, "namespace": "ai-tenant-e2e-mt-21b19766", "name": "openshift-service-ca.crt", "reconcileID": "ca9d05b9-9c59-4647-abde-a4f76cf99a48", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-21b19766"} 2026-07-09T04:43:35Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-21b19766"}, "namespace": "ai-tenant-e2e-mt-21b19766", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "00a06fc3-40b3-435e-a565-4e2e411d51d2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-21b19766"} 2026-07-09T04:43:35Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-21b19766"}, "namespace": "ai-tenant-e2e-mt-21b19766", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "00a06fc3-40b3-435e-a565-4e2e411d51d2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-21b19766"} 2026-07-09T04:43:35Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d1630ae3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3", "reconcileID": "90e5df70-7c46-4bae-8636-df78a2e6b974", "Gateway": "e2e-mt-d1630ae3", "namespace": "openshift-ingress"} 2026-07-09T04:43:35Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d1630ae3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3", "reconcileID": "90e5df70-7c46-4bae-8636-df78a2e6b974", "Gateway": "e2e-mt-d1630ae3", "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3-authn-ssl"} 2026-07-09T04:43:35Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d1630ae3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3", "reconcileID": "90e5df70-7c46-4bae-8636-df78a2e6b974", "Gateway": "e2e-mt-d1630ae3", "namespace": "openshift-ingress"} 2026-07-09T04:43:35Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d1630ae3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3", "reconcileID": "f56e8bea-8a3a-438a-9d58-fb28f6c879f6", "Gateway": "e2e-mt-d1630ae3", "namespace": "openshift-ingress"} 2026-07-09T04:43:35Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d1630ae3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3", "reconcileID": "f56e8bea-8a3a-438a-9d58-fb28f6c879f6", "Gateway": "e2e-mt-d1630ae3", "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3-authn-ssl"} 2026-07-09T04:43:35Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d1630ae3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3", "reconcileID": "f56e8bea-8a3a-438a-9d58-fb28f6c879f6", "Gateway": "e2e-mt-d1630ae3", "namespace": "openshift-ingress"} 2026-07-09T04:43:35Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d1630ae3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3", "reconcileID": "04c6086f-4c52-4fc2-80de-ddffd3ddad3d", "Gateway": "e2e-mt-d1630ae3", "namespace": "openshift-ingress"} 2026-07-09T04:43:35Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d1630ae3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3", "reconcileID": "04c6086f-4c52-4fc2-80de-ddffd3ddad3d", "Gateway": "e2e-mt-d1630ae3", "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3-authn-ssl"} 2026-07-09T04:43:35Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d1630ae3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3", "reconcileID": "04c6086f-4c52-4fc2-80de-ddffd3ddad3d", "Gateway": "e2e-mt-d1630ae3", "namespace": "openshift-ingress"} 2026-07-09T04:43:35Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d1630ae3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3", "reconcileID": "11f9e80f-d3c6-430d-85e2-0fc8a945cf57", "Gateway": "e2e-mt-d1630ae3", "namespace": "openshift-ingress"} 2026-07-09T04:43:35Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d1630ae3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3", "reconcileID": "11f9e80f-d3c6-430d-85e2-0fc8a945cf57", "Gateway": "e2e-mt-d1630ae3", "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3-authn-ssl"} 2026-07-09T04:43:35Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d1630ae3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3", "reconcileID": "11f9e80f-d3c6-430d-85e2-0fc8a945cf57", "Gateway": "e2e-mt-d1630ae3", "namespace": "openshift-ingress"} 2026-07-09T04:43:36Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d1630ae3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3", "reconcileID": "74bf6941-c5a2-4cbc-88e6-bf84e58b01d0", "Gateway": "e2e-mt-d1630ae3", "namespace": "openshift-ingress"} 2026-07-09T04:43:36Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d1630ae3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3", "reconcileID": "74bf6941-c5a2-4cbc-88e6-bf84e58b01d0", "Gateway": "e2e-mt-d1630ae3", "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3-authn-ssl"} 2026-07-09T04:43:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d1630ae3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3", "reconcileID": "74bf6941-c5a2-4cbc-88e6-bf84e58b01d0", "Gateway": "e2e-mt-d1630ae3", "namespace": "openshift-ingress"} 2026-07-09T04:43:36Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d1630ae3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3", "reconcileID": "fbc246f1-b83c-47ab-9c71-037969d3071d", "Gateway": "e2e-mt-d1630ae3", "namespace": "openshift-ingress"} 2026-07-09T04:43:36Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d1630ae3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3", "reconcileID": "fbc246f1-b83c-47ab-9c71-037969d3071d", "Gateway": "e2e-mt-d1630ae3", "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3-authn-ssl"} 2026-07-09T04:43:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d1630ae3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3", "reconcileID": "fbc246f1-b83c-47ab-9c71-037969d3071d", "Gateway": "e2e-mt-d1630ae3", "namespace": "openshift-ingress"} 2026-07-09T04:43:36Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d1630ae3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3", "reconcileID": "c6631c09-e5d1-4cf1-ac4a-987c727766b0", "Gateway": "e2e-mt-d1630ae3", "namespace": "openshift-ingress"} 2026-07-09T04:43:36Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d1630ae3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3", "reconcileID": "c6631c09-e5d1-4cf1-ac4a-987c727766b0", "Gateway": "e2e-mt-d1630ae3", "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3-authn-ssl"} 2026-07-09T04:43:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d1630ae3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3", "reconcileID": "c6631c09-e5d1-4cf1-ac4a-987c727766b0", "Gateway": "e2e-mt-d1630ae3", "namespace": "openshift-ingress"} 2026-07-09T04:46:43Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-d1630ae3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-d1630ae3", "reconcileID": "55008715-e99b-46a0-87d1-e4291ad2b2ac", "Gateway": "e2e-mt-d1630ae3", "namespace": "openshift-ingress"} 2026-07-09T04:46:43Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-4d5885e0"}, "namespace": "e2e-webhook-sub-4d5885e0", "name": "openshift-service-ca.crt", "reconcileID": "4839b4b6-7bbf-46c7-a1e2-ccab2597d759", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-4d5885e0"} 2026-07-09T04:46:43Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-4d5885e0"}, "namespace": "e2e-webhook-sub-4d5885e0", "name": "odh-trusted-ca-bundle", "reconcileID": "40c871a7-68d4-4fad-a755-1651046c3433", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-4d5885e0"} 2026-07-09T04:46:43Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-4d5885e0"}, "namespace": "e2e-webhook-sub-4d5885e0", "name": "openshift-service-ca.crt", "reconcileID": "8f2f36a2-5f27-4202-a526-a84ec48a4494", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-4d5885e0", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:46:43Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-4d5885e0"}, "namespace": "e2e-webhook-sub-4d5885e0", "name": "openshift-service-ca.crt", "reconcileID": "8f2f36a2-5f27-4202-a526-a84ec48a4494", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-4d5885e0"} 2026-07-09T04:46:43Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-4d5885e0"}, "namespace": "e2e-webhook-sub-4d5885e0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d521548d-b8aa-4c4e-9372-690cc7f2782e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-4d5885e0"} 2026-07-09T04:46:43Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-4d5885e0"}, "namespace": "e2e-webhook-sub-4d5885e0", "name": "openshift-service-ca.crt", "reconcileID": "97b9a7f1-ffa8-44c3-9c9d-a2cee869caae", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-4d5885e0"} 2026-07-09T04:46:43Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-4d5885e0"}, "namespace": "e2e-webhook-sub-4d5885e0", "name": "odh-trusted-ca-bundle", "reconcileID": "ecd800f3-5c59-4015-bbaa-5e6cdc0a0809", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-4d5885e0", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:46:43Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-4d5885e0"}, "namespace": "e2e-webhook-sub-4d5885e0", "name": "odh-trusted-ca-bundle", "reconcileID": "ecd800f3-5c59-4015-bbaa-5e6cdc0a0809", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-4d5885e0"} 2026-07-09T04:46:43Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-4d5885e0"}, "namespace": "e2e-webhook-sub-4d5885e0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "19e47795-3f60-4c69-abb7-bff66a33f7cf", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-4d5885e0"} 2026-07-09T04:46:49Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-4d5885e0"}, "namespace": "e2e-webhook-sub-4d5885e0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cb3b5370-c7f5-418e-8d80-36f2b22fc8a7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-4d5885e0"} 2026-07-09T04:46:49Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-4d5885e0"}, "namespace": "e2e-webhook-sub-4d5885e0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cb3b5370-c7f5-418e-8d80-36f2b22fc8a7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-4d5885e0", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:46:49Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-4d5885e0"}, "namespace": "e2e-webhook-sub-4d5885e0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cb3b5370-c7f5-418e-8d80-36f2b22fc8a7", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-sub-4d5885e0 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T04:46:49Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-4d5885e0"}, "namespace": "e2e-webhook-sub-4d5885e0", "name": "odh-trusted-ca-bundle", "reconcileID": "691d9300-c3e7-4c4b-b7c8-aa7876fa6937", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-4d5885e0"} 2026-07-09T04:46:49Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-sub-4d5885e0"}, "namespace": "e2e-webhook-sub-4d5885e0", "name": "odh-trusted-ca-bundle", "reconcileID": "691d9300-c3e7-4c4b-b7c8-aa7876fa6937", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-sub-4d5885e0"} 2026-07-09T04:46:49Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-4d5885e0"}, "namespace": "e2e-webhook-sub-4d5885e0", "name": "openshift-service-ca.crt", "reconcileID": "9f9aaf96-ebeb-4d96-80d0-5064a2e1f4d5", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-4d5885e0"} 2026-07-09T04:46:49Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-sub-4d5885e0"}, "namespace": "e2e-webhook-sub-4d5885e0", "name": "openshift-service-ca.crt", "reconcileID": "9f9aaf96-ebeb-4d96-80d0-5064a2e1f4d5", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-sub-4d5885e0"} 2026-07-09T04:46:49Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-4d5885e0"}, "namespace": "e2e-webhook-sub-4d5885e0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8b915227-34c3-4cc7-94a0-c99c17912ca6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-4d5885e0"} 2026-07-09T04:46:49Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-sub-4d5885e0"}, "namespace": "e2e-webhook-sub-4d5885e0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8b915227-34c3-4cc7-94a0-c99c17912ca6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-sub-4d5885e0"} 2026-07-09T04:46:49Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-99ed73d9"}, "namespace": "e2e-webhook-auth-99ed73d9", "name": "openshift-service-ca.crt", "reconcileID": "c550323b-4fdd-4a7d-b0a5-9b16fc403a00", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-99ed73d9"} 2026-07-09T04:46:49Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-99ed73d9"}, "namespace": "e2e-webhook-auth-99ed73d9", "name": "odh-trusted-ca-bundle", "reconcileID": "f539e1a9-c999-423d-9993-6daa1917519c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-99ed73d9"} 2026-07-09T04:46:49Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-99ed73d9"}, "namespace": "e2e-webhook-auth-99ed73d9", "name": "openshift-service-ca.crt", "reconcileID": "509f7160-b4e2-4c28-afaf-4335cb3c5e5d", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-99ed73d9", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:46:49Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-99ed73d9"}, "namespace": "e2e-webhook-auth-99ed73d9", "name": "openshift-service-ca.crt", "reconcileID": "509f7160-b4e2-4c28-afaf-4335cb3c5e5d", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-99ed73d9"} 2026-07-09T04:46:49Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-99ed73d9"}, "namespace": "e2e-webhook-auth-99ed73d9", "name": "openshift-service-ca.crt", "reconcileID": "e087a4a8-0387-4ba5-8c80-10b84cd0aa42", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-99ed73d9"} 2026-07-09T04:46:49Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-99ed73d9"}, "namespace": "e2e-webhook-auth-99ed73d9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b444feeb-4006-48e3-ae8e-60f6a4e11114", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-99ed73d9"} 2026-07-09T04:46:49Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-99ed73d9"}, "namespace": "e2e-webhook-auth-99ed73d9", "name": "odh-trusted-ca-bundle", "reconcileID": "4d184f88-f70e-483a-a040-1327b6dc837c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-99ed73d9", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:46:49Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-99ed73d9"}, "namespace": "e2e-webhook-auth-99ed73d9", "name": "odh-trusted-ca-bundle", "reconcileID": "4d184f88-f70e-483a-a040-1327b6dc837c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-99ed73d9"} 2026-07-09T04:46:49Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-99ed73d9"}, "namespace": "e2e-webhook-auth-99ed73d9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3d8b304c-61c5-4bbf-b62e-bb0b345b6727", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-99ed73d9"} 2026-07-09T04:46:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-99ed73d9"}, "namespace": "e2e-webhook-auth-99ed73d9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b50e1ca4-e316-4a67-90d2-c3e8ba813560", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-99ed73d9"} 2026-07-09T04:46:55Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-99ed73d9"}, "namespace": "e2e-webhook-auth-99ed73d9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b50e1ca4-e316-4a67-90d2-c3e8ba813560", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-99ed73d9", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:46:55Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-99ed73d9"}, "namespace": "e2e-webhook-auth-99ed73d9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b50e1ca4-e316-4a67-90d2-c3e8ba813560", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-webhook-auth-99ed73d9 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T04:46:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-99ed73d9"}, "namespace": "e2e-webhook-auth-99ed73d9", "name": "odh-trusted-ca-bundle", "reconcileID": "efb955a8-bd97-4f7e-9fc1-5457e9261247", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-99ed73d9"} 2026-07-09T04:46:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-99ed73d9"}, "namespace": "e2e-webhook-auth-99ed73d9", "name": "odh-trusted-ca-bundle", "reconcileID": "efb955a8-bd97-4f7e-9fc1-5457e9261247", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-99ed73d9"} 2026-07-09T04:46:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-99ed73d9"}, "namespace": "e2e-webhook-auth-99ed73d9", "name": "openshift-service-ca.crt", "reconcileID": "4cde6fb9-3196-4bef-b277-1e6f407dc8a7", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-99ed73d9"} 2026-07-09T04:46:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-99ed73d9"}, "namespace": "e2e-webhook-auth-99ed73d9", "name": "openshift-service-ca.crt", "reconcileID": "4cde6fb9-3196-4bef-b277-1e6f407dc8a7", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-99ed73d9"} 2026-07-09T04:46:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-99ed73d9"}, "namespace": "e2e-webhook-auth-99ed73d9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "67392bd9-ec11-4d9b-bb1b-0cbd1ab2b724", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-99ed73d9"} 2026-07-09T04:46:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-99ed73d9"}, "namespace": "e2e-webhook-auth-99ed73d9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "67392bd9-ec11-4d9b-bb1b-0cbd1ab2b724", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-99ed73d9"} 2026-07-09T04:47:02Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f266a62c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c", "reconcileID": "82e23ab2-e4bc-4cab-ae02-3cf8bb6c2fc9", "Gateway": "e2e-mt-f266a62c", "namespace": "openshift-ingress"} 2026-07-09T04:47:02Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f266a62c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c", "reconcileID": "82e23ab2-e4bc-4cab-ae02-3cf8bb6c2fc9", "Gateway": "e2e-mt-f266a62c", "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c-authn-ssl"} 2026-07-09T04:47:02Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f266a62c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c", "reconcileID": "82e23ab2-e4bc-4cab-ae02-3cf8bb6c2fc9", "Gateway": "e2e-mt-f266a62c", "namespace": "openshift-ingress"} 2026-07-09T04:47:02Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f266a62c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c", "reconcileID": "ca56fb7d-bec4-4bef-8ecd-c03bd299be20", "Gateway": "e2e-mt-f266a62c", "namespace": "openshift-ingress"} 2026-07-09T04:47:02Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f266a62c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c", "reconcileID": "ca56fb7d-bec4-4bef-8ecd-c03bd299be20", "Gateway": "e2e-mt-f266a62c", "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c-authn-ssl"} 2026-07-09T04:47:02Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f266a62c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c", "reconcileID": "ca56fb7d-bec4-4bef-8ecd-c03bd299be20", "Gateway": "e2e-mt-f266a62c", "namespace": "openshift-ingress"} 2026-07-09T04:47:02Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f266a62c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c", "reconcileID": "10d8417c-3d0c-41d7-bfbd-2471e13b661a", "Gateway": "e2e-mt-f266a62c", "namespace": "openshift-ingress"} 2026-07-09T04:47:02Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f266a62c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c", "reconcileID": "10d8417c-3d0c-41d7-bfbd-2471e13b661a", "Gateway": "e2e-mt-f266a62c", "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c-authn-ssl"} 2026-07-09T04:47:02Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f266a62c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c", "reconcileID": "10d8417c-3d0c-41d7-bfbd-2471e13b661a", "Gateway": "e2e-mt-f266a62c", "namespace": "openshift-ingress"} 2026-07-09T04:47:02Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f266a62c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c", "reconcileID": "709f5a57-b8be-4290-a474-a58d2d03434f", "Gateway": "e2e-mt-f266a62c", "namespace": "openshift-ingress"} 2026-07-09T04:47:02Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f266a62c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c", "reconcileID": "709f5a57-b8be-4290-a474-a58d2d03434f", "Gateway": "e2e-mt-f266a62c", "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c-authn-ssl"} 2026-07-09T04:47:02Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f266a62c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c", "reconcileID": "709f5a57-b8be-4290-a474-a58d2d03434f", "Gateway": "e2e-mt-f266a62c", "namespace": "openshift-ingress"} 2026-07-09T04:47:02Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f266a62c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c", "reconcileID": "978011ba-3a75-405e-81ed-d8fc332e1839", "Gateway": "e2e-mt-f266a62c", "namespace": "openshift-ingress"} 2026-07-09T04:47:02Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f266a62c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c", "reconcileID": "978011ba-3a75-405e-81ed-d8fc332e1839", "Gateway": "e2e-mt-f266a62c", "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c-authn-ssl"} 2026-07-09T04:47:02Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f266a62c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c", "reconcileID": "978011ba-3a75-405e-81ed-d8fc332e1839", "Gateway": "e2e-mt-f266a62c", "namespace": "openshift-ingress"} 2026-07-09T04:47:02Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f266a62c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c", "reconcileID": "680535ff-f6d5-422d-8053-f7e89c025dd3", "Gateway": "e2e-mt-f266a62c", "namespace": "openshift-ingress"} 2026-07-09T04:47:02Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f266a62c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c", "reconcileID": "680535ff-f6d5-422d-8053-f7e89c025dd3", "Gateway": "e2e-mt-f266a62c", "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c-authn-ssl"} 2026-07-09T04:47:02Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f266a62c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c", "reconcileID": "680535ff-f6d5-422d-8053-f7e89c025dd3", "Gateway": "e2e-mt-f266a62c", "namespace": "openshift-ingress"} 2026-07-09T04:47:04Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f266a62c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c", "reconcileID": "a7d88a23-6c3f-4fba-903c-f4307ea01c42", "Gateway": "e2e-mt-f266a62c", "namespace": "openshift-ingress"} 2026-07-09T04:47:04Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f266a62c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c", "reconcileID": "a7d88a23-6c3f-4fba-903c-f4307ea01c42", "Gateway": "e2e-mt-f266a62c", "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c-authn-ssl"} 2026-07-09T04:47:04Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f266a62c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c", "reconcileID": "a7d88a23-6c3f-4fba-903c-f4307ea01c42", "Gateway": "e2e-mt-f266a62c", "namespace": "openshift-ingress"} 2026-07-09T04:50:09Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-f266a62c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-f266a62c", "reconcileID": "e457f87b-b74b-4328-8dbc-423aad65ef42", "Gateway": "e2e-mt-f266a62c", "namespace": "openshift-ingress"} 2026-07-09T04:50:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-335a752d"}, "namespace": "ai-tenant-e2e-mt-335a752d", "name": "openshift-service-ca.crt", "reconcileID": "0e68c328-c774-4f69-b3a7-96e4a78ef5fd", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-335a752d"} 2026-07-09T04:50:11Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-335a752d"}, "namespace": "ai-tenant-e2e-mt-335a752d", "name": "openshift-service-ca.crt", "reconcileID": "c8b3423a-56a7-409b-80c3-e573c277cb19", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-335a752d", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:50:11Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-335a752d"}, "namespace": "ai-tenant-e2e-mt-335a752d", "name": "openshift-service-ca.crt", "reconcileID": "c8b3423a-56a7-409b-80c3-e573c277cb19", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-335a752d"} 2026-07-09T04:50:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-335a752d"}, "namespace": "ai-tenant-e2e-mt-335a752d", "name": "odh-trusted-ca-bundle", "reconcileID": "f8dae5c0-3abf-463b-9b16-1bb04896679a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-335a752d"} 2026-07-09T04:50:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-335a752d"}, "namespace": "ai-tenant-e2e-mt-335a752d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "23908159-b1c2-40c2-9bd2-71db825486e1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-335a752d"} 2026-07-09T04:50:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-335a752d"}, "namespace": "ai-tenant-e2e-mt-335a752d", "name": "openshift-service-ca.crt", "reconcileID": "04b10694-206c-44b0-ae92-6929589b6e10", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-335a752d"} 2026-07-09T04:50:11Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-335a752d"}, "namespace": "ai-tenant-e2e-mt-335a752d", "name": "odh-trusted-ca-bundle", "reconcileID": "4d7ce652-c5cb-4bc3-b600-a3c8221d5b9c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-335a752d", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:50:11Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-335a752d"}, "namespace": "ai-tenant-e2e-mt-335a752d", "name": "odh-trusted-ca-bundle", "reconcileID": "4d7ce652-c5cb-4bc3-b600-a3c8221d5b9c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-335a752d"} 2026-07-09T04:50:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-335a752d"}, "namespace": "ai-tenant-e2e-mt-335a752d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "01200495-1197-487b-9bb2-98024931d864", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-335a752d"} 2026-07-09T04:50:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-b1ceed98"}, "namespace": "ai-tenant-e2e-mt-b1ceed98", "name": "openshift-service-ca.crt", "reconcileID": "ecde434b-e17c-406e-9aa4-6a72c9bb7aca", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-b1ceed98"} 2026-07-09T04:50:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-b1ceed98"}, "namespace": "ai-tenant-e2e-mt-b1ceed98", "name": "odh-trusted-ca-bundle", "reconcileID": "ee49c25b-ca62-4f96-87e3-0c29b8a59663", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-b1ceed98"} 2026-07-09T04:50:11Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-b1ceed98"}, "namespace": "ai-tenant-e2e-mt-b1ceed98", "name": "openshift-service-ca.crt", "reconcileID": "bca6b1f7-f158-4c7c-b963-7a4cbb010b10", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-b1ceed98", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:50:11Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-b1ceed98"}, "namespace": "ai-tenant-e2e-mt-b1ceed98", "name": "openshift-service-ca.crt", "reconcileID": "bca6b1f7-f158-4c7c-b963-7a4cbb010b10", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-b1ceed98"} 2026-07-09T04:50:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-b1ceed98"}, "namespace": "ai-tenant-e2e-mt-b1ceed98", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d79050c8-1bf7-447b-a663-956561b419fb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-b1ceed98"} 2026-07-09T04:50:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-b1ceed98"}, "namespace": "ai-tenant-e2e-mt-b1ceed98", "name": "openshift-service-ca.crt", "reconcileID": "259a1485-6c8e-47b7-ae8e-db6806f36b59", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-b1ceed98"} 2026-07-09T04:50:11Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-b1ceed98"}, "namespace": "ai-tenant-e2e-mt-b1ceed98", "name": "odh-trusted-ca-bundle", "reconcileID": "bff73895-2329-4989-8eb2-ce41fb3d9a4c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-b1ceed98", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:50:11Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-b1ceed98"}, "namespace": "ai-tenant-e2e-mt-b1ceed98", "name": "odh-trusted-ca-bundle", "reconcileID": "bff73895-2329-4989-8eb2-ce41fb3d9a4c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-b1ceed98"} 2026-07-09T04:50:11Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-b1ceed98"}, "namespace": "ai-tenant-e2e-mt-b1ceed98", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8c701bce-d7ca-457f-9bf7-edd7166dec4a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-b1ceed98"} 2026-07-09T04:50:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-335a752d"}, "namespace": "ai-tenant-e2e-mt-335a752d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a6dc518d-d40c-48b6-a452-97be2897c808", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-335a752d"} 2026-07-09T04:50:19Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-335a752d"}, "namespace": "ai-tenant-e2e-mt-335a752d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a6dc518d-d40c-48b6-a452-97be2897c808", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-335a752d", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:50:19Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-335a752d"}, "namespace": "ai-tenant-e2e-mt-335a752d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a6dc518d-d40c-48b6-a452-97be2897c808", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-335a752d because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T04:50:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-335a752d"}, "namespace": "ai-tenant-e2e-mt-335a752d", "name": "odh-trusted-ca-bundle", "reconcileID": "c78c5eac-548d-4333-be68-d7b4a1dfe0ff", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-335a752d"} 2026-07-09T04:50:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-335a752d"}, "namespace": "ai-tenant-e2e-mt-335a752d", "name": "odh-trusted-ca-bundle", "reconcileID": "c78c5eac-548d-4333-be68-d7b4a1dfe0ff", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-335a752d"} 2026-07-09T04:50:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-335a752d"}, "namespace": "ai-tenant-e2e-mt-335a752d", "name": "openshift-service-ca.crt", "reconcileID": "518c0177-d27f-4396-addf-1682987ddd2f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-335a752d"} 2026-07-09T04:50:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-335a752d"}, "namespace": "ai-tenant-e2e-mt-335a752d", "name": "openshift-service-ca.crt", "reconcileID": "518c0177-d27f-4396-addf-1682987ddd2f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-335a752d"} 2026-07-09T04:50:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-335a752d"}, "namespace": "ai-tenant-e2e-mt-335a752d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ae3ca132-5ed6-4835-a063-080b6a29e1b0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-335a752d"} 2026-07-09T04:50:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-335a752d"}, "namespace": "ai-tenant-e2e-mt-335a752d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ae3ca132-5ed6-4835-a063-080b6a29e1b0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-335a752d"} 2026-07-09T04:50:25Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-b1ceed98"}, "namespace": "ai-tenant-e2e-mt-b1ceed98", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6a7c4d4e-44ae-457d-bfea-a27fd2f20263", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-b1ceed98"} 2026-07-09T04:50:25Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-b1ceed98"}, "namespace": "ai-tenant-e2e-mt-b1ceed98", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6a7c4d4e-44ae-457d-bfea-a27fd2f20263", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-b1ceed98", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:50:25Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-b1ceed98"}, "namespace": "ai-tenant-e2e-mt-b1ceed98", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6a7c4d4e-44ae-457d-bfea-a27fd2f20263", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-b1ceed98 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T04:50:25Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-b1ceed98"}, "namespace": "ai-tenant-e2e-mt-b1ceed98", "name": "odh-trusted-ca-bundle", "reconcileID": "5761a588-d839-42d5-813e-32e2a5a16d15", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-b1ceed98"} 2026-07-09T04:50:25Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-b1ceed98"}, "namespace": "ai-tenant-e2e-mt-b1ceed98", "name": "odh-trusted-ca-bundle", "reconcileID": "5761a588-d839-42d5-813e-32e2a5a16d15", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-b1ceed98"} 2026-07-09T04:50:25Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-b1ceed98"}, "namespace": "ai-tenant-e2e-mt-b1ceed98", "name": "openshift-service-ca.crt", "reconcileID": "39294b63-18ff-4680-85aa-992775ae937d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-b1ceed98"} 2026-07-09T04:50:25Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-b1ceed98"}, "namespace": "ai-tenant-e2e-mt-b1ceed98", "name": "openshift-service-ca.crt", "reconcileID": "39294b63-18ff-4680-85aa-992775ae937d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-b1ceed98"} 2026-07-09T04:50:25Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-b1ceed98"}, "namespace": "ai-tenant-e2e-mt-b1ceed98", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "60edb88e-9ade-4fad-8289-f5712e567730", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-b1ceed98"} 2026-07-09T04:50:25Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-b1ceed98"}, "namespace": "ai-tenant-e2e-mt-b1ceed98", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "60edb88e-9ade-4fad-8289-f5712e567730", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-b1ceed98"} 2026-07-09T04:50:25Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-38c49105"}, "namespace": "ai-tenant-e2e-mt-38c49105", "name": "openshift-service-ca.crt", "reconcileID": "54ab4dab-a80c-4881-a0d3-6f8f7d5b0bee", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-38c49105"} 2026-07-09T04:50:25Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-38c49105"}, "namespace": "ai-tenant-e2e-mt-38c49105", "name": "odh-trusted-ca-bundle", "reconcileID": "131ddfdc-fc89-4ebc-af7b-e64c6a966885", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-38c49105"} 2026-07-09T04:50:25Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-38c49105"}, "namespace": "ai-tenant-e2e-mt-38c49105", "name": "openshift-service-ca.crt", "reconcileID": "d96bd4b2-c38f-4611-a5ab-3acb25f77f76", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-38c49105", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:50:25Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-38c49105"}, "namespace": "ai-tenant-e2e-mt-38c49105", "name": "openshift-service-ca.crt", "reconcileID": "d96bd4b2-c38f-4611-a5ab-3acb25f77f76", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-38c49105"} 2026-07-09T04:50:25Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-38c49105"}, "namespace": "ai-tenant-e2e-mt-38c49105", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9e1667d6-f716-4ddc-82b1-17d5a57d3224", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-38c49105"} 2026-07-09T04:50:25Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-38c49105"}, "namespace": "ai-tenant-e2e-mt-38c49105", "name": "openshift-service-ca.crt", "reconcileID": "3893c285-809b-43dd-9930-ceab3505f004", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-38c49105"} 2026-07-09T04:50:25Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-38c49105"}, "namespace": "ai-tenant-e2e-mt-38c49105", "name": "odh-trusted-ca-bundle", "reconcileID": "df4e1a7d-d100-4583-b90f-a1ad0dd93328", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-38c49105", "update": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:50:25Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-38c49105"}, "namespace": "ai-tenant-e2e-mt-38c49105", "name": "odh-trusted-ca-bundle", "reconcileID": "df4e1a7d-d100-4583-b90f-a1ad0dd93328", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-38c49105"} 2026-07-09T04:50:25Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-38c49105"}, "namespace": "ai-tenant-e2e-mt-38c49105", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "86cd8841-1d3c-4048-be5b-cbcf7267ba4d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-38c49105"} 2026-07-09T04:51:02Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-38c49105"}, "namespace": "ai-tenant-e2e-mt-38c49105", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5c57a362-89de-43bc-8930-2de9053fdf5f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-38c49105"} 2026-07-09T04:51:02Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-38c49105"}, "namespace": "ai-tenant-e2e-mt-38c49105", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5c57a362-89de-43bc-8930-2de9053fdf5f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-38c49105", "create": "odh-kserve-custom-ca-bundle"} 2026-07-09T04:51:02Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-38c49105"}, "namespace": "ai-tenant-e2e-mt-38c49105", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5c57a362-89de-43bc-8930-2de9053fdf5f", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-38c49105 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-09T04:51:02Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-38c49105"}, "namespace": "ai-tenant-e2e-mt-38c49105", "name": "odh-trusted-ca-bundle", "reconcileID": "cebf85a3-c106-4165-b78b-0d3db130cb4f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-38c49105"} 2026-07-09T04:51:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-38c49105"}, "namespace": "ai-tenant-e2e-mt-38c49105", "name": "odh-trusted-ca-bundle", "reconcileID": "cebf85a3-c106-4165-b78b-0d3db130cb4f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-38c49105"} 2026-07-09T04:51:02Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-38c49105"}, "namespace": "ai-tenant-e2e-mt-38c49105", "name": "openshift-service-ca.crt", "reconcileID": "9d9bd446-8db3-4c48-9c8e-239250528ae7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-38c49105"} 2026-07-09T04:51:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-38c49105"}, "namespace": "ai-tenant-e2e-mt-38c49105", "name": "openshift-service-ca.crt", "reconcileID": "9d9bd446-8db3-4c48-9c8e-239250528ae7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-38c49105"} 2026-07-09T04:51:02Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-38c49105"}, "namespace": "ai-tenant-e2e-mt-38c49105", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "abfb5d05-fcbb-4ec1-aa1b-5fb6fcffa40d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-38c49105"} 2026-07-09T04:51:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-38c49105"}, "namespace": "ai-tenant-e2e-mt-38c49105", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "abfb5d05-fcbb-4ec1-aa1b-5fb6fcffa40d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-38c49105"} 2026-07-09T04:51:03Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-4a3e5d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d", "reconcileID": "bc730764-4a76-45f6-87d4-ea606fe09ca1", "Gateway": "e2e-inf-a-4a3e5d", "namespace": "openshift-ingress"} 2026-07-09T04:51:03Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-4a3e5d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d", "reconcileID": "bc730764-4a76-45f6-87d4-ea606fe09ca1", "Gateway": "e2e-inf-a-4a3e5d", "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d-authn-ssl"} 2026-07-09T04:51:03Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-4a3e5d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d", "reconcileID": "bc730764-4a76-45f6-87d4-ea606fe09ca1", "Gateway": "e2e-inf-a-4a3e5d", "namespace": "openshift-ingress"} 2026-07-09T04:51:03Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-4a3e5d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d", "reconcileID": "459691ef-06c7-41dc-8f7a-65f7a0283fcc", "Gateway": "e2e-inf-a-4a3e5d", "namespace": "openshift-ingress"} 2026-07-09T04:51:03Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-4a3e5d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d", "reconcileID": "459691ef-06c7-41dc-8f7a-65f7a0283fcc", "Gateway": "e2e-inf-a-4a3e5d", "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d-authn-ssl"} 2026-07-09T04:51:03Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-4a3e5d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d", "reconcileID": "459691ef-06c7-41dc-8f7a-65f7a0283fcc", "Gateway": "e2e-inf-a-4a3e5d", "namespace": "openshift-ingress"} 2026-07-09T04:51:03Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-4a3e5d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d", "reconcileID": "59e6ad6f-bd97-4ef2-aff1-f9d5fb0866a2", "Gateway": "e2e-inf-a-4a3e5d", "namespace": "openshift-ingress"} 2026-07-09T04:51:03Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-4a3e5d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d", "reconcileID": "59e6ad6f-bd97-4ef2-aff1-f9d5fb0866a2", "Gateway": "e2e-inf-a-4a3e5d", "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d-authn-ssl"} 2026-07-09T04:51:03Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-4a3e5d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d", "reconcileID": "59e6ad6f-bd97-4ef2-aff1-f9d5fb0866a2", "Gateway": "e2e-inf-a-4a3e5d", "namespace": "openshift-ingress"} 2026-07-09T04:51:03Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-4a3e5d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d", "reconcileID": "23096797-e0ad-427d-abcf-a3329d4bd8f8", "Gateway": "e2e-inf-a-4a3e5d", "namespace": "openshift-ingress"} 2026-07-09T04:51:03Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-4a3e5d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d", "reconcileID": "23096797-e0ad-427d-abcf-a3329d4bd8f8", "Gateway": "e2e-inf-a-4a3e5d", "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d-authn-ssl"} 2026-07-09T04:51:03Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-4a3e5d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d", "reconcileID": "23096797-e0ad-427d-abcf-a3329d4bd8f8", "Gateway": "e2e-inf-a-4a3e5d", "namespace": "openshift-ingress"} 2026-07-09T04:51:03Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-4a3e5d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d", "reconcileID": "5a4d45ea-9ed6-41f4-87dd-ddb89ed7a578", "Gateway": "e2e-inf-a-4a3e5d", "namespace": "openshift-ingress"} 2026-07-09T04:51:03Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-4a3e5d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d", "reconcileID": "5a4d45ea-9ed6-41f4-87dd-ddb89ed7a578", "Gateway": "e2e-inf-a-4a3e5d", "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d-authn-ssl"} 2026-07-09T04:51:03Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-4a3e5d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d", "reconcileID": "5a4d45ea-9ed6-41f4-87dd-ddb89ed7a578", "Gateway": "e2e-inf-a-4a3e5d", "namespace": "openshift-ingress"} 2026-07-09T04:51:04Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-4a3e5d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d", "reconcileID": "c30837b5-ad54-4704-837e-ca70e0c6a488", "Gateway": "e2e-inf-a-4a3e5d", "namespace": "openshift-ingress"} 2026-07-09T04:51:04Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-4a3e5d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d", "reconcileID": "c30837b5-ad54-4704-837e-ca70e0c6a488", "Gateway": "e2e-inf-a-4a3e5d", "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d-authn-ssl"} 2026-07-09T04:51:04Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-4a3e5d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d", "reconcileID": "c30837b5-ad54-4704-837e-ca70e0c6a488", "Gateway": "e2e-inf-a-4a3e5d", "namespace": "openshift-ingress"} 2026-07-09T04:51:05Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-4a3e5d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d", "reconcileID": "816db9c3-55c7-45c3-bc90-0924b146c859", "Gateway": "e2e-inf-a-4a3e5d", "namespace": "openshift-ingress"} 2026-07-09T04:51:05Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-4a3e5d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d", "reconcileID": "816db9c3-55c7-45c3-bc90-0924b146c859", "Gateway": "e2e-inf-a-4a3e5d", "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d-authn-ssl"} 2026-07-09T04:51:05Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-4a3e5d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d", "reconcileID": "816db9c3-55c7-45c3-bc90-0924b146c859", "Gateway": "e2e-inf-a-4a3e5d", "namespace": "openshift-ingress"} 2026-07-09T04:54:08Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-09T04:54:08Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-09T04:54:10Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-09T04:54:10Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-09T04:54:11Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-4a3e5d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-4a3e5d", "reconcileID": "15786f8b-2385-4c68-9f17-d4293c4c59f5", "Gateway": "e2e-inf-a-4a3e5d", "namespace": "openshift-ingress"}