2026-07-13T04:10:22Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-42421d30"}, "namespace": "ai-tenant-e2e-mt-42421d30", "name": "odh-trusted-ca-bundle", "reconcileID": "e7cee545-6da6-4de9-88db-33a098ef3d47", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-42421d30"} 2026-07-13T04:10:22Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-42421d30"}, "namespace": "ai-tenant-e2e-mt-42421d30", "name": "odh-trusted-ca-bundle", "reconcileID": "7ad4c203-e5b2-4158-890b-0f41ea0d78da", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-42421d30", "update": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:10:22Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-42421d30"}, "namespace": "ai-tenant-e2e-mt-42421d30", "name": "odh-trusted-ca-bundle", "reconcileID": "7ad4c203-e5b2-4158-890b-0f41ea0d78da", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-42421d30"} 2026-07-13T04:10:22Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-42421d30"}, "namespace": "ai-tenant-e2e-mt-42421d30", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "44872644-990b-466b-9cc7-23e64dc5dda4", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-42421d30"} 2026-07-13T04:10:25Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-13T04:10:25Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-13T04:10:27Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-13T04:10:27Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-13T04:10:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-fe244c13"}, "namespace": "ai-tenant-e2e-mt-fe244c13", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cd75a7f3-5ea9-4d05-bacb-174ecb42df91", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-fe244c13"} 2026-07-13T04:10:29Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-fe244c13"}, "namespace": "ai-tenant-e2e-mt-fe244c13", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cd75a7f3-5ea9-4d05-bacb-174ecb42df91", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-fe244c13", "create": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:10:29Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-fe244c13"}, "namespace": "ai-tenant-e2e-mt-fe244c13", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cd75a7f3-5ea9-4d05-bacb-174ecb42df91", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-fe244c13 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-13T04:10:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-fe244c13"}, "namespace": "ai-tenant-e2e-mt-fe244c13", "name": "odh-trusted-ca-bundle", "reconcileID": "f56c80b8-7981-46b8-9fd6-2890f5bf3cea", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-fe244c13"} 2026-07-13T04:10:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-fe244c13"}, "namespace": "ai-tenant-e2e-mt-fe244c13", "name": "odh-trusted-ca-bundle", "reconcileID": "f56c80b8-7981-46b8-9fd6-2890f5bf3cea", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-fe244c13"} 2026-07-13T04:10:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-fe244c13"}, "namespace": "ai-tenant-e2e-mt-fe244c13", "name": "openshift-service-ca.crt", "reconcileID": "02687e8a-3715-462d-9bc3-a3c7e8e73726", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-fe244c13"} 2026-07-13T04:10:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-fe244c13"}, "namespace": "ai-tenant-e2e-mt-fe244c13", "name": "openshift-service-ca.crt", "reconcileID": "02687e8a-3715-462d-9bc3-a3c7e8e73726", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-fe244c13"} 2026-07-13T04:10:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-fe244c13"}, "namespace": "ai-tenant-e2e-mt-fe244c13", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "431b6ff5-e60d-444e-a8dd-6cb4b7d8b4ce", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-fe244c13"} 2026-07-13T04:10:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-fe244c13"}, "namespace": "ai-tenant-e2e-mt-fe244c13", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "431b6ff5-e60d-444e-a8dd-6cb4b7d8b4ce", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-fe244c13"} 2026-07-13T04:10:36Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-42421d30"}, "namespace": "ai-tenant-e2e-mt-42421d30", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "18dfebcb-7d37-415e-a423-56d406963a2d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-42421d30"} 2026-07-13T04:10:36Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-42421d30"}, "namespace": "ai-tenant-e2e-mt-42421d30", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "18dfebcb-7d37-415e-a423-56d406963a2d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-42421d30", "create": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:10:36Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-42421d30"}, "namespace": "ai-tenant-e2e-mt-42421d30", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "18dfebcb-7d37-415e-a423-56d406963a2d", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-42421d30 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-13T04:10:36Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-42421d30"}, "namespace": "ai-tenant-e2e-mt-42421d30", "name": "odh-trusted-ca-bundle", "reconcileID": "36b6c9b7-6679-426e-8d81-37646d042e8f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-42421d30"} 2026-07-13T04:10:36Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-42421d30"}, "namespace": "ai-tenant-e2e-mt-42421d30", "name": "odh-trusted-ca-bundle", "reconcileID": "36b6c9b7-6679-426e-8d81-37646d042e8f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-42421d30"} 2026-07-13T04:10:36Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-42421d30"}, "namespace": "ai-tenant-e2e-mt-42421d30", "name": "openshift-service-ca.crt", "reconcileID": "7ac2468e-8184-41f6-a826-35c08ccaf06c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-42421d30"} 2026-07-13T04:10:36Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-42421d30"}, "namespace": "ai-tenant-e2e-mt-42421d30", "name": "openshift-service-ca.crt", "reconcileID": "7ac2468e-8184-41f6-a826-35c08ccaf06c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-42421d30"} 2026-07-13T04:10:36Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-42421d30"}, "namespace": "ai-tenant-e2e-mt-42421d30", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "23c98f00-7e9e-493c-8854-39b9eaed4a0d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-42421d30"} 2026-07-13T04:10:36Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-42421d30"}, "namespace": "ai-tenant-e2e-mt-42421d30", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "23c98f00-7e9e-493c-8854-39b9eaed4a0d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-42421d30"} 2026-07-13T04:10:36Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-4d71fb1a"}, "namespace": "ai-tenant-e2e-mt-4d71fb1a", "name": "openshift-service-ca.crt", "reconcileID": "45ccf20a-f5cd-452f-ad2d-7d452483063e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-4d71fb1a"} 2026-07-13T04:10:36Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-4d71fb1a"}, "namespace": "ai-tenant-e2e-mt-4d71fb1a", "name": "openshift-service-ca.crt", "reconcileID": "5661ea57-98d4-4449-acd7-c69d8cc6ed9c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-4d71fb1a", "create": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:10:36Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-4d71fb1a"}, "namespace": "ai-tenant-e2e-mt-4d71fb1a", "name": "openshift-service-ca.crt", "reconcileID": "5661ea57-98d4-4449-acd7-c69d8cc6ed9c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-4d71fb1a"} 2026-07-13T04:10:36Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-4d71fb1a"}, "namespace": "ai-tenant-e2e-mt-4d71fb1a", "name": "odh-trusted-ca-bundle", "reconcileID": "8204bf20-e09e-40e7-8cfc-5d06e75e250d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-4d71fb1a"} 2026-07-13T04:10:36Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-4d71fb1a"}, "namespace": "ai-tenant-e2e-mt-4d71fb1a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e6f03a32-6604-4efd-b7fe-df703d974cc8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-4d71fb1a"} 2026-07-13T04:10:36Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-4d71fb1a"}, "namespace": "ai-tenant-e2e-mt-4d71fb1a", "name": "openshift-service-ca.crt", "reconcileID": "c976c03d-08ce-4c26-af39-90e1a0318ad9", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-4d71fb1a"} 2026-07-13T04:10:36Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-4d71fb1a"}, "namespace": "ai-tenant-e2e-mt-4d71fb1a", "name": "odh-trusted-ca-bundle", "reconcileID": "fe91cbe1-d95a-4fc5-86fe-cc4fe7cadcdd", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-4d71fb1a", "update": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:10:36Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-4d71fb1a"}, "namespace": "ai-tenant-e2e-mt-4d71fb1a", "name": "odh-trusted-ca-bundle", "reconcileID": "fe91cbe1-d95a-4fc5-86fe-cc4fe7cadcdd", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-4d71fb1a"} 2026-07-13T04:10:36Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-4d71fb1a"}, "namespace": "ai-tenant-e2e-mt-4d71fb1a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "da7885e2-0e48-4a44-abe5-cb84559014a2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-4d71fb1a"} 2026-07-13T04:11:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-4d71fb1a"}, "namespace": "ai-tenant-e2e-mt-4d71fb1a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "af8ff541-2bb1-4b81-b1fd-ae2a7099483b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-4d71fb1a"} 2026-07-13T04:11:13Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-4d71fb1a"}, "namespace": "ai-tenant-e2e-mt-4d71fb1a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "af8ff541-2bb1-4b81-b1fd-ae2a7099483b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-4d71fb1a", "create": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:11:13Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-4d71fb1a"}, "namespace": "ai-tenant-e2e-mt-4d71fb1a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "af8ff541-2bb1-4b81-b1fd-ae2a7099483b", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-4d71fb1a because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-13T04:11:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-4d71fb1a"}, "namespace": "ai-tenant-e2e-mt-4d71fb1a", "name": "odh-trusted-ca-bundle", "reconcileID": "48728706-99d6-49db-86be-fa43cb78915a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-4d71fb1a"} 2026-07-13T04:11:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-4d71fb1a"}, "namespace": "ai-tenant-e2e-mt-4d71fb1a", "name": "odh-trusted-ca-bundle", "reconcileID": "48728706-99d6-49db-86be-fa43cb78915a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-4d71fb1a"} 2026-07-13T04:11:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-4d71fb1a"}, "namespace": "ai-tenant-e2e-mt-4d71fb1a", "name": "openshift-service-ca.crt", "reconcileID": "00964324-64ef-4794-80b5-43b62e56fe6b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-4d71fb1a"} 2026-07-13T04:11:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-4d71fb1a"}, "namespace": "ai-tenant-e2e-mt-4d71fb1a", "name": "openshift-service-ca.crt", "reconcileID": "00964324-64ef-4794-80b5-43b62e56fe6b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-4d71fb1a"} 2026-07-13T04:11:13Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-4d71fb1a"}, "namespace": "ai-tenant-e2e-mt-4d71fb1a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4bb18df7-ee72-4923-8060-0eacc8de8b16", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-4d71fb1a"} 2026-07-13T04:11:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-4d71fb1a"}, "namespace": "ai-tenant-e2e-mt-4d71fb1a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4bb18df7-ee72-4923-8060-0eacc8de8b16", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-4d71fb1a"} 2026-07-13T04:11:14Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "23ceb3e0-04ab-408a-ac27-2e2510d222ce", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:14Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "23ceb3e0-04ab-408a-ac27-2e2510d222ce", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3-authn-ssl"} 2026-07-13T04:11:14Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "23ceb3e0-04ab-408a-ac27-2e2510d222ce", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:14Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "3aa92e2e-2938-4949-93c7-f47c35d03d22", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:14Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "3aa92e2e-2938-4949-93c7-f47c35d03d22", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3-authn-ssl"} 2026-07-13T04:11:14Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "3aa92e2e-2938-4949-93c7-f47c35d03d22", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:14Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "48e88585-bf62-4071-b5e5-0bbcc9f912e9", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:14Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "48e88585-bf62-4071-b5e5-0bbcc9f912e9", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3-authn-ssl"} 2026-07-13T04:11:14Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "48e88585-bf62-4071-b5e5-0bbcc9f912e9", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:14Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "45fb5770-20b8-4f60-a549-7531d9b4d9da", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:14Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "45fb5770-20b8-4f60-a549-7531d9b4d9da", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3-authn-ssl"} 2026-07-13T04:11:14Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "45fb5770-20b8-4f60-a549-7531d9b4d9da", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:14Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "5cdd838d-098c-41f6-a1a1-de4fcadc3eb9", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:14Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "5cdd838d-098c-41f6-a1a1-de4fcadc3eb9", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3-authn-ssl"} 2026-07-13T04:11:14Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "5cdd838d-098c-41f6-a1a1-de4fcadc3eb9", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:14Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "ed16a7bf-7d0b-4222-be49-d7a6a5dfb9db", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:14Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "ed16a7bf-7d0b-4222-be49-d7a6a5dfb9db", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3-authn-ssl"} 2026-07-13T04:11:14Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "ed16a7bf-7d0b-4222-be49-d7a6a5dfb9db", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:15Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "c2da9875-f885-43d5-9009-00067625cd4c", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:15Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "c2da9875-f885-43d5-9009-00067625cd4c", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3-authn-ssl"} 2026-07-13T04:11:15Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "c2da9875-f885-43d5-9009-00067625cd4c", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "openshift-service-ca.crt", "reconcileID": "f154e987-2de9-4ce1-8342-4a0a8715463b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:11:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "odh-trusted-ca-bundle", "reconcileID": "6efb267e-f389-42ce-97c2-7ad9ff00e5bb", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:11:17Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "openshift-service-ca.crt", "reconcileID": "c76dd25b-cc5d-4c85-a919-933c23482db2", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-42c2d3", "create": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:11:17Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "openshift-service-ca.crt", "reconcileID": "c76dd25b-cc5d-4c85-a919-933c23482db2", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:11:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6aa0c393-1491-4bf5-b48e-3ce5bc648b2b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:11:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "openshift-service-ca.crt", "reconcileID": "5424a418-fd69-4b9f-8b85-d27065882d2b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:11:17Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "odh-trusted-ca-bundle", "reconcileID": "aa688b68-6cf2-4d16-9264-5890c2647850", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-42c2d3", "update": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:11:17Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "odh-trusted-ca-bundle", "reconcileID": "aa688b68-6cf2-4d16-9264-5890c2647850", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:11:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c7ed1d04-70bd-434b-b98b-8c2fdb1e77bf", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:11:17Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "29182442-b428-4185-a836-2656eed1ed20", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:17Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "29182442-b428-4185-a836-2656eed1ed20", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3-authn-ssl"} 2026-07-13T04:11:17Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "29182442-b428-4185-a836-2656eed1ed20", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "7e2e63c3-aa31-4c8c-b662-ccad9faafd8e", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-13T04:11:22Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "179f7bee-2620-40f8-97b3-bb282d724e02", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:11:22Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "179f7bee-2620-40f8-97b3-bb282d724e02", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a-authn-ssl"} 2026-07-13T04:11:22Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "179f7bee-2620-40f8-97b3-bb282d724e02", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:11:22Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "5ee0592b-b163-4462-96c0-4b131fbbcb12", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:11:22Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "5ee0592b-b163-4462-96c0-4b131fbbcb12", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a-authn-ssl"} 2026-07-13T04:11:22Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "5ee0592b-b163-4462-96c0-4b131fbbcb12", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:11:22Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "b9f87336-fca3-45b3-ba0a-b41bcb306163", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:11:22Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "b9f87336-fca3-45b3-ba0a-b41bcb306163", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a-authn-ssl"} 2026-07-13T04:11:22Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "b9f87336-fca3-45b3-ba0a-b41bcb306163", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:11:22Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "729f6038-ee46-4d7b-b91b-8cf0c5999bab", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:11:22Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "729f6038-ee46-4d7b-b91b-8cf0c5999bab", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a-authn-ssl"} 2026-07-13T04:11:22Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "729f6038-ee46-4d7b-b91b-8cf0c5999bab", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:11:22Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "85e49fe4-ae2b-4254-b8cb-22598b70218b", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:11:22Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "85e49fe4-ae2b-4254-b8cb-22598b70218b", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a-authn-ssl"} 2026-07-13T04:11:22Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "85e49fe4-ae2b-4254-b8cb-22598b70218b", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:11:22Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "ae857655-702f-4c9e-adee-061fce45991c", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:11:22Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "ae857655-702f-4c9e-adee-061fce45991c", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a-authn-ssl"} 2026-07-13T04:11:22Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "ae857655-702f-4c9e-adee-061fce45991c", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:11:23Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "e1cc6bf1-2adf-4638-b593-0c4c9479d9c5", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:11:23Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "e1cc6bf1-2adf-4638-b593-0c4c9479d9c5", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a-authn-ssl"} 2026-07-13T04:11:23Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "e1cc6bf1-2adf-4638-b593-0c4c9479d9c5", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:11:24Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "78c98c44-b798-4697-88ef-77de7efad115", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:24Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "78c98c44-b798-4697-88ef-77de7efad115", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3-authn-ssl"} 2026-07-13T04:11:24Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "78c98c44-b798-4697-88ef-77de7efad115", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:25Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "openshift-service-ca.crt", "reconcileID": "bd0bf222-41d1-4abf-8a25-702064d59ca4", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:11:25Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "odh-trusted-ca-bundle", "reconcileID": "869aaed3-6285-4084-aa8f-d808b253acbd", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:11:25Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "openshift-service-ca.crt", "reconcileID": "f50c0a8a-98c8-467b-ad81-1af6a8eeb7b3", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-88358a", "create": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:11:25Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "openshift-service-ca.crt", "reconcileID": "f50c0a8a-98c8-467b-ad81-1af6a8eeb7b3", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:11:25Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "openshift-service-ca.crt", "reconcileID": "03f145e0-b267-43e1-87a5-dda977894d84", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:11:25Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "822bfc7f-552e-412c-ae40-f78d126fba81", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:11:25Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "odh-trusted-ca-bundle", "reconcileID": "3196a071-ddc5-440f-91f6-63d9042eca54", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-88358a", "update": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:11:25Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "odh-trusted-ca-bundle", "reconcileID": "3196a071-ddc5-440f-91f6-63d9042eca54", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:11:25Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e57de4ea-638e-48fd-90e4-bdb4cfa9d2d0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:11:25Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "dcf64df2-5969-4d32-8f41-d2b9f8e52094", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:11:25Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "dcf64df2-5969-4d32-8f41-d2b9f8e52094", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a-authn-ssl"} 2026-07-13T04:11:25Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "dcf64df2-5969-4d32-8f41-d2b9f8e52094", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:11:25Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "37fd758f-ec9f-4328-b374-66a62058be51", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-13T04:11:29Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "a375ea18-267e-4ca2-951f-9da6b9f77715", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:11:29Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "a375ea18-267e-4ca2-951f-9da6b9f77715", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:11:29Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "a375ea18-267e-4ca2-951f-9da6b9f77715", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:11:29Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "a66277db-3ce7-454f-bf0e-02beb93a6039", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:29Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "a375ea18-267e-4ca2-951f-9da6b9f77715", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3-kserve-route-authn"} 2026-07-13T04:11:29Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "a375ea18-267e-4ca2-951f-9da6b9f77715", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3", "authType": "userdefined"} 2026-07-13T04:11:29Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "a375ea18-267e-4ca2-951f-9da6b9f77715", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:11:29Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "a66277db-3ce7-454f-bf0e-02beb93a6039", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3-authn-ssl"} 2026-07-13T04:11:29Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "a66277db-3ce7-454f-bf0e-02beb93a6039", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:29Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "167a8fbf-5133-419a-8f21-45d1a937cd70", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:11:29Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "167a8fbf-5133-419a-8f21-45d1a937cd70", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:11:29Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "167a8fbf-5133-419a-8f21-45d1a937cd70", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:11:29Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "167a8fbf-5133-419a-8f21-45d1a937cd70", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3-kserve-route-authn"} 2026-07-13T04:11:29Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "167a8fbf-5133-419a-8f21-45d1a937cd70", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3", "authType": "userdefined"} 2026-07-13T04:11:29Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "167a8fbf-5133-419a-8f21-45d1a937cd70", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:11:31Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "6ec56c15-6408-427d-85ca-02c304949a71", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:11:31Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "6ec56c15-6408-427d-85ca-02c304949a71", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a-authn-ssl"} 2026-07-13T04:11:31Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "6ec56c15-6408-427d-85ca-02c304949a71", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:11:38Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "117d4443-eb01-484f-a166-5668cbe84a3b", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:38Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "117d4443-eb01-484f-a166-5668cbe84a3b", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3-authn-ssl"} 2026-07-13T04:11:38Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "117d4443-eb01-484f-a166-5668cbe84a3b", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:39Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "306c438d-befa-4b3b-a95d-98cc0ff71e65", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-13T04:11:39Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "315713dd-534d-48f0-af13-1c75da5c2f93", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:11:39Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "315713dd-534d-48f0-af13-1c75da5c2f93", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:11:39Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "315713dd-534d-48f0-af13-1c75da5c2f93", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:11:39Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "315713dd-534d-48f0-af13-1c75da5c2f93", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3-kserve-route-authn"} 2026-07-13T04:11:39Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "315713dd-534d-48f0-af13-1c75da5c2f93", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3", "authType": "userdefined"} 2026-07-13T04:11:39Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "315713dd-534d-48f0-af13-1c75da5c2f93", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:11:46Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "6b8be19e-1b2f-46e7-a45f-3b662760fa58", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:46Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "6b8be19e-1b2f-46e7-a45f-3b662760fa58", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3-authn-ssl"} 2026-07-13T04:11:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "6b8be19e-1b2f-46e7-a45f-3b662760fa58", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:11:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "356de38e-35c8-437c-b25c-f957ced03ef4", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-13T04:11:51Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "56722dfa-eb04-4d6b-8620-5b7cf9839524", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:11:51Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "56722dfa-eb04-4d6b-8620-5b7cf9839524", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:11:51Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "56722dfa-eb04-4d6b-8620-5b7cf9839524", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:11:51Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "56722dfa-eb04-4d6b-8620-5b7cf9839524", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3-kserve-route-authn"} 2026-07-13T04:11:51Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "56722dfa-eb04-4d6b-8620-5b7cf9839524", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3", "authType": "userdefined"} 2026-07-13T04:11:51Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "56722dfa-eb04-4d6b-8620-5b7cf9839524", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:11:56Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "25ebd105-46a5-4169-b933-0724dd99c3ac", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:11:56Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "25ebd105-46a5-4169-b933-0724dd99c3ac", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:11:56Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "25ebd105-46a5-4169-b933-0724dd99c3ac", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:11:56Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "b52ca46b-dc3f-45c8-a154-41693cb1e891", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:11:56Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "25ebd105-46a5-4169-b933-0724dd99c3ac", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a-kserve-route-authn"} 2026-07-13T04:11:56Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "25ebd105-46a5-4169-b933-0724dd99c3ac", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a", "authType": "userdefined"} 2026-07-13T04:11:56Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "25ebd105-46a5-4169-b933-0724dd99c3ac", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:11:56Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "b52ca46b-dc3f-45c8-a154-41693cb1e891", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a-authn-ssl"} 2026-07-13T04:11:56Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "b52ca46b-dc3f-45c8-a154-41693cb1e891", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:11:56Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "8e3573ec-76b9-4200-8c9a-e95e49aeddd5", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:11:56Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "8e3573ec-76b9-4200-8c9a-e95e49aeddd5", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:11:56Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "8e3573ec-76b9-4200-8c9a-e95e49aeddd5", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:11:56Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "8e3573ec-76b9-4200-8c9a-e95e49aeddd5", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a-kserve-route-authn"} 2026-07-13T04:11:56Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "8e3573ec-76b9-4200-8c9a-e95e49aeddd5", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a", "authType": "userdefined"} 2026-07-13T04:11:56Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "8e3573ec-76b9-4200-8c9a-e95e49aeddd5", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:12:16Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "6e4befd7-ba0c-48fb-a54a-3c90d22a66c2", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:12:16Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "6e4befd7-ba0c-48fb-a54a-3c90d22a66c2", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a-authn-ssl"} 2026-07-13T04:12:16Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "6e4befd7-ba0c-48fb-a54a-3c90d22a66c2", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:12:17Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "62c2d0ed-f681-4008-8542-c54ee9eaa2d7", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:12:17Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "62c2d0ed-f681-4008-8542-c54ee9eaa2d7", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:12:17Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "62c2d0ed-f681-4008-8542-c54ee9eaa2d7", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:12:17Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "62c2d0ed-f681-4008-8542-c54ee9eaa2d7", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a-kserve-route-authn"} 2026-07-13T04:12:17Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "62c2d0ed-f681-4008-8542-c54ee9eaa2d7", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a", "authType": "userdefined"} 2026-07-13T04:12:17Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "62c2d0ed-f681-4008-8542-c54ee9eaa2d7", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:12:22Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "d88e78c4-ae21-474c-9ffc-d0d9a2276df0", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:12:22Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "d88e78c4-ae21-474c-9ffc-d0d9a2276df0", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a-authn-ssl"} 2026-07-13T04:12:22Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "d88e78c4-ae21-474c-9ffc-d0d9a2276df0", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:12:27Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "01b6762b-5c99-4281-9138-9ffdae9004df", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:12:27Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "01b6762b-5c99-4281-9138-9ffdae9004df", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:12:27Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "01b6762b-5c99-4281-9138-9ffdae9004df", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:12:27Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "01b6762b-5c99-4281-9138-9ffdae9004df", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a-kserve-route-authn"} 2026-07-13T04:12:27Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "01b6762b-5c99-4281-9138-9ffdae9004df", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a", "authType": "userdefined"} 2026-07-13T04:12:27Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "01b6762b-5c99-4281-9138-9ffdae9004df", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:12:38Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "f8f4d862-0bf4-4135-a608-55db33ce934b", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:12:38Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "f8f4d862-0bf4-4135-a608-55db33ce934b", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3"} 2026-07-13T04:12:38Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "f8f4d862-0bf4-4135-a608-55db33ce934b", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:12:38Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "f8f4d862-0bf4-4135-a608-55db33ce934b", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:12:38Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "f8f4d862-0bf4-4135-a608-55db33ce934b", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3-kserve-route-authn"} 2026-07-13T04:12:38Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "f8f4d862-0bf4-4135-a608-55db33ce934b", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3-kserve-route-authn"} 2026-07-13T04:12:38Z DEBUG Triggering LLMInferenceService Cleanup for Namespace {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "f8f4d862-0bf4-4135-a608-55db33ce934b", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:12:38Z DEBUG Cleaning up LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "f8f4d862-0bf4-4135-a608-55db33ce934b", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:12:38Z INFO Stop LLMInferenceService reconciliation {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-42c2d3","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "test-model-42c2d3", "reconcileID": "4a2899f4-dc06-46fd-ac85-5b67a7d3d835", "LLMInferenceService": "test-model-42c2d3", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:12:38Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "c968c363-9f7f-4292-8a4a-a4e3ab45ab6d", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:12:38Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "c968c363-9f7f-4292-8a4a-a4e3ab45ab6d", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3-authn-ssl"} 2026-07-13T04:12:38Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "c968c363-9f7f-4292-8a4a-a4e3ab45ab6d", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:12:38Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "88114ef1-95eb-4723-8e1e-48a66b66827c", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:12:38Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "88114ef1-95eb-4723-8e1e-48a66b66827c", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3-authn-ssl"} 2026-07-13T04:12:38Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "88114ef1-95eb-4723-8e1e-48a66b66827c", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:12:39Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "56520085-a6e5-4f94-a601-468964f73c63", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:12:39Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "56520085-a6e5-4f94-a601-468964f73c63", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a"} 2026-07-13T04:12:39Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "56520085-a6e5-4f94-a601-468964f73c63", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:12:39Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "56520085-a6e5-4f94-a601-468964f73c63", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:12:39Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "56520085-a6e5-4f94-a601-468964f73c63", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a-kserve-route-authn"} 2026-07-13T04:12:39Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "56520085-a6e5-4f94-a601-468964f73c63", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a-kserve-route-authn"} 2026-07-13T04:12:39Z DEBUG Triggering LLMInferenceService Cleanup for Namespace {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "56520085-a6e5-4f94-a601-468964f73c63", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:12:39Z DEBUG Cleaning up LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "56520085-a6e5-4f94-a601-468964f73c63", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:12:39Z INFO Stop LLMInferenceService reconciliation {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-88358a","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "test-model-88358a", "reconcileID": "35c5e9d1-4bae-4f82-8069-18da23c5f0e4", "LLMInferenceService": "test-model-88358a", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:12:39Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "450f35f7-9840-4af3-93c5-b2f6e345814d", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:12:39Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "450f35f7-9840-4af3-93c5-b2f6e345814d", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a-authn-ssl"} 2026-07-13T04:12:39Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "450f35f7-9840-4af3-93c5-b2f6e345814d", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:12:39Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "77dda97a-d3a1-4289-a546-681009895ccc", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:12:39Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "77dda97a-d3a1-4289-a546-681009895ccc", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a-authn-ssl"} 2026-07-13T04:12:39Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "77dda97a-d3a1-4289-a546-681009895ccc", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:12:39Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "9371a2a5-df59-4c60-b91b-1acc6d77d97c", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-13T04:12:40Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "dedc36d3-2a52-48dc-b102-012990bb3ce1", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:12:40Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "dedc36d3-2a52-48dc-b102-012990bb3ce1", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3-authn-ssl"} 2026-07-13T04:12:40Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-42c2d3","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-42c2d3", "reconcileID": "dedc36d3-2a52-48dc-b102-012990bb3ce1", "Gateway": "e2e-inf-a-42c2d3", "namespace": "openshift-ingress"} 2026-07-13T04:13:21Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8278b169-44bf-4d88-b0eb-2e894c92eca7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:13:21Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8278b169-44bf-4d88-b0eb-2e894c92eca7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-42c2d3", "create": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:13:21Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8278b169-44bf-4d88-b0eb-2e894c92eca7", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-inf-a-42c2d3 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-13T04:13:21Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "odh-trusted-ca-bundle", "reconcileID": "8a9a89d4-6644-4856-81e0-28a58c730b6e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:13:21Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "odh-trusted-ca-bundle", "reconcileID": "8a9a89d4-6644-4856-81e0-28a58c730b6e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:13:21Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "openshift-service-ca.crt", "reconcileID": "e69f262e-bd6d-4e80-b630-9f05993b4f4b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:13:21Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "openshift-service-ca.crt", "reconcileID": "e69f262e-bd6d-4e80-b630-9f05993b4f4b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:13:21Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c8e0d886-bf34-4884-8ffd-a05a56e434db", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:13:21Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-42c2d3"}, "namespace": "ai-tenant-e2e-inf-a-42c2d3", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c8e0d886-bf34-4884-8ffd-a05a56e434db", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-42c2d3"} 2026-07-13T04:13:22Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "7810afec-c143-46cf-a865-54224ec6cc13", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-13T04:13:23Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "5b3c3790-ebfa-417a-a029-a3b0b66f8e66", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:13:23Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "5b3c3790-ebfa-417a-a029-a3b0b66f8e66", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a-authn-ssl"} 2026-07-13T04:13:23Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-88358a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-88358a", "reconcileID": "5b3c3790-ebfa-417a-a029-a3b0b66f8e66", "Gateway": "e2e-inf-b-88358a", "namespace": "openshift-ingress"} 2026-07-13T04:13:28Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a01e73e4-0203-42aa-83a4-69557d09cf99", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:13:28Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a01e73e4-0203-42aa-83a4-69557d09cf99", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-88358a", "create": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:13:28Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a01e73e4-0203-42aa-83a4-69557d09cf99", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-inf-b-88358a because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-13T04:13:28Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "odh-trusted-ca-bundle", "reconcileID": "e7007822-2585-472b-9a67-c41a6d7c79ed", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:13:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "odh-trusted-ca-bundle", "reconcileID": "e7007822-2585-472b-9a67-c41a6d7c79ed", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:13:28Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "openshift-service-ca.crt", "reconcileID": "76bf578d-96b0-4487-a4bf-438f2d683492", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:13:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "openshift-service-ca.crt", "reconcileID": "76bf578d-96b0-4487-a4bf-438f2d683492", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:13:28Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c6a987f3-bda6-4d32-a6cb-4db7ccccb588", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:13:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-88358a"}, "namespace": "ai-tenant-e2e-inf-b-88358a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c6a987f3-bda6-4d32-a6cb-4db7ccccb588", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-88358a"} 2026-07-13T04:13:29Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "2812638b-d1f8-4920-90d7-1f6e2a8f4f89", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress"} 2026-07-13T04:13:29Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "2812638b-d1f8-4920-90d7-1f6e2a8f4f89", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc-authn-ssl"} 2026-07-13T04:13:29Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "2812638b-d1f8-4920-90d7-1f6e2a8f4f89", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress"} 2026-07-13T04:13:29Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "8d5b3922-bfb2-4055-99e1-f5841d19fe01", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress"} 2026-07-13T04:13:29Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "8d5b3922-bfb2-4055-99e1-f5841d19fe01", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc-authn-ssl"} 2026-07-13T04:13:29Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "8d5b3922-bfb2-4055-99e1-f5841d19fe01", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress"} 2026-07-13T04:13:29Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "8f54dbeb-8e46-47b1-9da4-54bea656aaca", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress"} 2026-07-13T04:13:29Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "8f54dbeb-8e46-47b1-9da4-54bea656aaca", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc-authn-ssl"} 2026-07-13T04:13:29Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "8f54dbeb-8e46-47b1-9da4-54bea656aaca", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress"} 2026-07-13T04:13:29Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "eb217ae4-cbe5-4f41-8d92-c578a7b5ffd0", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress"} 2026-07-13T04:13:29Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "eb217ae4-cbe5-4f41-8d92-c578a7b5ffd0", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc-authn-ssl"} 2026-07-13T04:13:29Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "eb217ae4-cbe5-4f41-8d92-c578a7b5ffd0", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress"} 2026-07-13T04:13:30Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "a9b9814a-7bad-4320-b383-0a9bd01e34ba", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress"} 2026-07-13T04:13:30Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "a9b9814a-7bad-4320-b383-0a9bd01e34ba", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc-authn-ssl"} 2026-07-13T04:13:30Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "a9b9814a-7bad-4320-b383-0a9bd01e34ba", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress"} 2026-07-13T04:13:31Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "67e286ec-a72f-4757-bc65-86dfc9195b87", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress"} 2026-07-13T04:13:31Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "67e286ec-a72f-4757-bc65-86dfc9195b87", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc-authn-ssl"} 2026-07-13T04:13:31Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "67e286ec-a72f-4757-bc65-86dfc9195b87", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress"} 2026-07-13T04:13:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-5e78fc"}, "namespace": "ai-tenant-e2e-api-a-5e78fc", "name": "openshift-service-ca.crt", "reconcileID": "c49680f7-4496-44d2-ac96-8646d1b75195", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-5e78fc"} 2026-07-13T04:13:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-5e78fc"}, "namespace": "ai-tenant-e2e-api-a-5e78fc", "name": "odh-trusted-ca-bundle", "reconcileID": "7be82117-050e-4da7-8912-6525fe378680", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-5e78fc"} 2026-07-13T04:13:32Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-5e78fc"}, "namespace": "ai-tenant-e2e-api-a-5e78fc", "name": "openshift-service-ca.crt", "reconcileID": "21ddd210-7272-4e6a-aac7-9b9b153dbb55", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-5e78fc", "create": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:13:32Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-5e78fc"}, "namespace": "ai-tenant-e2e-api-a-5e78fc", "name": "openshift-service-ca.crt", "reconcileID": "21ddd210-7272-4e6a-aac7-9b9b153dbb55", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-5e78fc"} 2026-07-13T04:13:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-5e78fc"}, "namespace": "ai-tenant-e2e-api-a-5e78fc", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9e76d591-3451-41fb-b645-441af5957c52", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-5e78fc"} 2026-07-13T04:13:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-5e78fc"}, "namespace": "ai-tenant-e2e-api-a-5e78fc", "name": "openshift-service-ca.crt", "reconcileID": "a8697f08-38a6-4dab-adf3-31e83bd604f7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-5e78fc"} 2026-07-13T04:13:33Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-5e78fc"}, "namespace": "ai-tenant-e2e-api-a-5e78fc", "name": "odh-trusted-ca-bundle", "reconcileID": "d16f9356-feef-40ed-bd7d-5551402b3f45", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-5e78fc", "update": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:13:33Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-5e78fc"}, "namespace": "ai-tenant-e2e-api-a-5e78fc", "name": "odh-trusted-ca-bundle", "reconcileID": "d16f9356-feef-40ed-bd7d-5551402b3f45", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-5e78fc"} 2026-07-13T04:13:33Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-5e78fc"}, "namespace": "ai-tenant-e2e-api-a-5e78fc", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "866c3734-54dd-4c5e-b134-5f7549fb6c30", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-5e78fc"} 2026-07-13T04:13:33Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "9a44270a-da8a-49ec-8cd8-fdd74d382314", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress"} 2026-07-13T04:13:33Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "9a44270a-da8a-49ec-8cd8-fdd74d382314", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc-authn-ssl"} 2026-07-13T04:13:33Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "9a44270a-da8a-49ec-8cd8-fdd74d382314", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress"} 2026-07-13T04:13:33Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "7a810a95-e9da-4af4-90b8-fabc8ada8ac0", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-13T04:13:37Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "69b4fc9b-19be-4c50-bad7-f260398369ff", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress"} 2026-07-13T04:13:38Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "69b4fc9b-19be-4c50-bad7-f260398369ff", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a-authn-ssl"} 2026-07-13T04:13:38Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "69b4fc9b-19be-4c50-bad7-f260398369ff", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress"} 2026-07-13T04:13:38Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "e886ef8c-dec5-453f-852b-83b99ead6584", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress"} 2026-07-13T04:13:38Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "e886ef8c-dec5-453f-852b-83b99ead6584", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a-authn-ssl"} 2026-07-13T04:13:38Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "e886ef8c-dec5-453f-852b-83b99ead6584", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress"} 2026-07-13T04:13:38Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "6d32da9f-4554-415a-9096-18eca02cc6e6", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress"} 2026-07-13T04:13:38Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "6d32da9f-4554-415a-9096-18eca02cc6e6", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a-authn-ssl"} 2026-07-13T04:13:38Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "6d32da9f-4554-415a-9096-18eca02cc6e6", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress"} 2026-07-13T04:13:38Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "80852443-53bb-4c1f-816b-aaaaae6489b0", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress"} 2026-07-13T04:13:38Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "80852443-53bb-4c1f-816b-aaaaae6489b0", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a-authn-ssl"} 2026-07-13T04:13:38Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "80852443-53bb-4c1f-816b-aaaaae6489b0", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress"} 2026-07-13T04:13:38Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "796ea0fc-aa13-418e-a37d-ef577f4da824", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress"} 2026-07-13T04:13:38Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "796ea0fc-aa13-418e-a37d-ef577f4da824", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a-authn-ssl"} 2026-07-13T04:13:38Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "796ea0fc-aa13-418e-a37d-ef577f4da824", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress"} 2026-07-13T04:13:38Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "772f9e40-8242-4f9d-8407-31d849e86745", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress"} 2026-07-13T04:13:38Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "772f9e40-8242-4f9d-8407-31d849e86745", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a-authn-ssl"} 2026-07-13T04:13:38Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "772f9e40-8242-4f9d-8407-31d849e86745", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress"} 2026-07-13T04:13:39Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "3f2cc435-bb79-437b-92cc-08bc332c54e1", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress"} 2026-07-13T04:13:39Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "3f2cc435-bb79-437b-92cc-08bc332c54e1", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a-authn-ssl"} 2026-07-13T04:13:39Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "3f2cc435-bb79-437b-92cc-08bc332c54e1", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress"} 2026-07-13T04:13:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-b-5c775a"}, "namespace": "ai-tenant-e2e-api-b-5c775a", "name": "openshift-service-ca.crt", "reconcileID": "5a082496-5b47-428b-8235-53da7159928d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-b-5c775a"} 2026-07-13T04:13:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-b-5c775a"}, "namespace": "ai-tenant-e2e-api-b-5c775a", "name": "odh-trusted-ca-bundle", "reconcileID": "ef20cd9c-c0e2-480f-a45e-5b6d40cfd1a7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-b-5c775a"} 2026-07-13T04:13:40Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-b-5c775a"}, "namespace": "ai-tenant-e2e-api-b-5c775a", "name": "openshift-service-ca.crt", "reconcileID": "4b308aac-664d-4def-9ba3-12635e2c108c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-b-5c775a", "create": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:13:40Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-b-5c775a"}, "namespace": "ai-tenant-e2e-api-b-5c775a", "name": "openshift-service-ca.crt", "reconcileID": "4b308aac-664d-4def-9ba3-12635e2c108c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-b-5c775a"} 2026-07-13T04:13:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-5c775a"}, "namespace": "ai-tenant-e2e-api-b-5c775a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ee0caa48-a041-49bc-9b4e-c3b500ba8884", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-b-5c775a"} 2026-07-13T04:13:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-b-5c775a"}, "namespace": "ai-tenant-e2e-api-b-5c775a", "name": "openshift-service-ca.crt", "reconcileID": "850185cd-92bc-4994-a721-bd4f013ca962", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-b-5c775a"} 2026-07-13T04:13:41Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-b-5c775a"}, "namespace": "ai-tenant-e2e-api-b-5c775a", "name": "odh-trusted-ca-bundle", "reconcileID": "fc8351e6-7014-4d27-909f-68a9ba0c0734", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-b-5c775a", "update": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:13:41Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-b-5c775a"}, "namespace": "ai-tenant-e2e-api-b-5c775a", "name": "odh-trusted-ca-bundle", "reconcileID": "fc8351e6-7014-4d27-909f-68a9ba0c0734", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-b-5c775a"} 2026-07-13T04:13:41Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-5c775a"}, "namespace": "ai-tenant-e2e-api-b-5c775a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a7cf2f23-b458-4076-80ed-d033ef0ecb84", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-b-5c775a"} 2026-07-13T04:13:41Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "0a96c790-4644-4e8b-9e68-3d483de1a8bb", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress"} 2026-07-13T04:13:41Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "0a96c790-4644-4e8b-9e68-3d483de1a8bb", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a-authn-ssl"} 2026-07-13T04:13:41Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "0a96c790-4644-4e8b-9e68-3d483de1a8bb", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress"} 2026-07-13T04:13:41Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "dd02e207-041e-448d-bb9c-ca800cf2f898", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-13T04:13:41Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "ff9aa387-3f2e-4256-a088-b0694ca78f23", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress"} 2026-07-13T04:13:41Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "ff9aa387-3f2e-4256-a088-b0694ca78f23", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc-authn-ssl"} 2026-07-13T04:13:41Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "ff9aa387-3f2e-4256-a088-b0694ca78f23", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress"} 2026-07-13T04:13:50Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "2eb28556-423c-4f0a-bc31-12f363a23ae9", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress"} 2026-07-13T04:13:50Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "2eb28556-423c-4f0a-bc31-12f363a23ae9", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a-authn-ssl"} 2026-07-13T04:13:50Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "2eb28556-423c-4f0a-bc31-12f363a23ae9", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress"} 2026-07-13T04:13:54Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "77561bb6-028c-4da6-b3f4-7420c23ef566", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-13T04:13:54Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "b3438938-8537-4a23-9615-ab55c5900c21", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress"} 2026-07-13T04:13:54Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "b3438938-8537-4a23-9615-ab55c5900c21", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc-authn-ssl"} 2026-07-13T04:13:54Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-5e78fc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-5e78fc", "reconcileID": "b3438938-8537-4a23-9615-ab55c5900c21", "Gateway": "e2e-api-a-5e78fc", "namespace": "openshift-ingress"} 2026-07-13T04:13:59Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-5e78fc"}, "namespace": "ai-tenant-e2e-api-a-5e78fc", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "76e84ee7-2e66-4325-84f6-84a98f2a931f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-5e78fc"} 2026-07-13T04:13:59Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-5e78fc"}, "namespace": "ai-tenant-e2e-api-a-5e78fc", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "76e84ee7-2e66-4325-84f6-84a98f2a931f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-5e78fc", "create": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:13:59Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-5e78fc"}, "namespace": "ai-tenant-e2e-api-a-5e78fc", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "76e84ee7-2e66-4325-84f6-84a98f2a931f", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-api-a-5e78fc because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-13T04:13:59Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-5e78fc"}, "namespace": "ai-tenant-e2e-api-a-5e78fc", "name": "odh-trusted-ca-bundle", "reconcileID": "ae10eefc-d5b0-4bed-a081-920c2e2e000e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-5e78fc"} 2026-07-13T04:13:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-5e78fc"}, "namespace": "ai-tenant-e2e-api-a-5e78fc", "name": "odh-trusted-ca-bundle", "reconcileID": "ae10eefc-d5b0-4bed-a081-920c2e2e000e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-5e78fc"} 2026-07-13T04:13:59Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-5e78fc"}, "namespace": "ai-tenant-e2e-api-a-5e78fc", "name": "openshift-service-ca.crt", "reconcileID": "53c730ca-4759-4592-84bd-0958a947cc50", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-5e78fc"} 2026-07-13T04:13:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-5e78fc"}, "namespace": "ai-tenant-e2e-api-a-5e78fc", "name": "openshift-service-ca.crt", "reconcileID": "53c730ca-4759-4592-84bd-0958a947cc50", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-5e78fc"} 2026-07-13T04:13:59Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-5e78fc"}, "namespace": "ai-tenant-e2e-api-a-5e78fc", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c8194291-07d6-467b-b738-ac6500d1c86c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-5e78fc"} 2026-07-13T04:13:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-5e78fc"}, "namespace": "ai-tenant-e2e-api-a-5e78fc", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c8194291-07d6-467b-b738-ac6500d1c86c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-5e78fc"} 2026-07-13T04:14:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "0bb1a28c-39cd-46c4-95f0-e3503c1c0a08", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-13T04:14:00Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "754b8439-08bd-4bc4-bf42-c14777bc3086", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress"} 2026-07-13T04:14:00Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "754b8439-08bd-4bc4-bf42-c14777bc3086", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a-authn-ssl"} 2026-07-13T04:14:00Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-5c775a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-5c775a", "reconcileID": "754b8439-08bd-4bc4-bf42-c14777bc3086", "Gateway": "e2e-api-b-5c775a", "namespace": "openshift-ingress"} 2026-07-13T04:14:06Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-5c775a"}, "namespace": "ai-tenant-e2e-api-b-5c775a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cd0826e8-daf1-4145-b484-3900aa2ac696", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-b-5c775a"} 2026-07-13T04:14:06Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-5c775a"}, "namespace": "ai-tenant-e2e-api-b-5c775a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cd0826e8-daf1-4145-b484-3900aa2ac696", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-b-5c775a", "create": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:14:06Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-5c775a"}, "namespace": "ai-tenant-e2e-api-b-5c775a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cd0826e8-daf1-4145-b484-3900aa2ac696", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-api-b-5c775a because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-13T04:14:06Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-b-5c775a"}, "namespace": "ai-tenant-e2e-api-b-5c775a", "name": "odh-trusted-ca-bundle", "reconcileID": "91d89d07-c39d-425d-9770-a3c0c9741777", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-b-5c775a"} 2026-07-13T04:14:06Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-b-5c775a"}, "namespace": "ai-tenant-e2e-api-b-5c775a", "name": "odh-trusted-ca-bundle", "reconcileID": "91d89d07-c39d-425d-9770-a3c0c9741777", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-b-5c775a"} 2026-07-13T04:14:06Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-b-5c775a"}, "namespace": "ai-tenant-e2e-api-b-5c775a", "name": "openshift-service-ca.crt", "reconcileID": "dc18e6b3-0576-46be-b497-2ceec3566dad", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-b-5c775a"} 2026-07-13T04:14:06Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-b-5c775a"}, "namespace": "ai-tenant-e2e-api-b-5c775a", "name": "openshift-service-ca.crt", "reconcileID": "dc18e6b3-0576-46be-b497-2ceec3566dad", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-b-5c775a"} 2026-07-13T04:14:06Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-5c775a"}, "namespace": "ai-tenant-e2e-api-b-5c775a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "48c5f300-f1d9-4876-8099-466e7fadcd82", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-b-5c775a"} 2026-07-13T04:14:06Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-5c775a"}, "namespace": "ai-tenant-e2e-api-b-5c775a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "48c5f300-f1d9-4876-8099-466e7fadcd82", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-b-5c775a"} 2026-07-13T04:14:07Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "564efdc2-21ac-4df6-8867-2bc7afbc5254", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress"} 2026-07-13T04:14:07Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "564efdc2-21ac-4df6-8867-2bc7afbc5254", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02-authn-ssl"} 2026-07-13T04:14:07Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "564efdc2-21ac-4df6-8867-2bc7afbc5254", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress"} 2026-07-13T04:14:07Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "f0e6ff89-1475-4100-b41a-67b31e0b348d", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress"} 2026-07-13T04:14:07Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "f0e6ff89-1475-4100-b41a-67b31e0b348d", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02-authn-ssl"} 2026-07-13T04:14:07Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "f0e6ff89-1475-4100-b41a-67b31e0b348d", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress"} 2026-07-13T04:14:07Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "2c1be32b-7edc-497f-91e8-04c88f24a4b4", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress"} 2026-07-13T04:14:07Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "2c1be32b-7edc-497f-91e8-04c88f24a4b4", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02-authn-ssl"} 2026-07-13T04:14:07Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "2c1be32b-7edc-497f-91e8-04c88f24a4b4", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress"} 2026-07-13T04:14:07Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "65e5939f-f1f2-402b-9514-a2b607f3c2cd", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress"} 2026-07-13T04:14:07Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "65e5939f-f1f2-402b-9514-a2b607f3c2cd", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02-authn-ssl"} 2026-07-13T04:14:07Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "65e5939f-f1f2-402b-9514-a2b607f3c2cd", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress"} 2026-07-13T04:14:07Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "6edad954-fc66-4586-968d-4f18362c11fb", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress"} 2026-07-13T04:14:07Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "6edad954-fc66-4586-968d-4f18362c11fb", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02-authn-ssl"} 2026-07-13T04:14:07Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "6edad954-fc66-4586-968d-4f18362c11fb", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress"} 2026-07-13T04:14:07Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "6173012e-00e2-4f81-a5f6-b0bbaf88d5d9", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress"} 2026-07-13T04:14:07Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "6173012e-00e2-4f81-a5f6-b0bbaf88d5d9", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02-authn-ssl"} 2026-07-13T04:14:07Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "6173012e-00e2-4f81-a5f6-b0bbaf88d5d9", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress"} 2026-07-13T04:14:08Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "bc678c09-d8d6-4b6d-8bcb-d41435b9b90b", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress"} 2026-07-13T04:14:08Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "bc678c09-d8d6-4b6d-8bcb-d41435b9b90b", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02-authn-ssl"} 2026-07-13T04:14:08Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "bc678c09-d8d6-4b6d-8bcb-d41435b9b90b", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress"} 2026-07-13T04:14:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-694c02"}, "namespace": "ai-tenant-e2e-shared-a-694c02", "name": "openshift-service-ca.crt", "reconcileID": "cf6c6267-a937-40e5-840e-9914f0e5dd2f", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-a-694c02"} 2026-07-13T04:14:10Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-a-694c02"}, "namespace": "ai-tenant-e2e-shared-a-694c02", "name": "odh-trusted-ca-bundle", "reconcileID": "556b0115-53fe-4a99-aca3-c7149d9e2a9e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-694c02"} 2026-07-13T04:14:10Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-694c02"}, "namespace": "ai-tenant-e2e-shared-a-694c02", "name": "openshift-service-ca.crt", "reconcileID": "e7e563b5-d1a9-4b4a-a973-cc4fcd592828", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-a-694c02", "create": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:14:10Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-694c02"}, "namespace": "ai-tenant-e2e-shared-a-694c02", "name": "openshift-service-ca.crt", "reconcileID": "e7e563b5-d1a9-4b4a-a973-cc4fcd592828", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-a-694c02"} 2026-07-13T04:14:10Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-694c02"}, "namespace": "ai-tenant-e2e-shared-a-694c02", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "42bce6dc-3874-4ddf-a95e-efced8a3e551", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-694c02"} 2026-07-13T04:14:10Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-694c02"}, "namespace": "ai-tenant-e2e-shared-a-694c02", "name": "openshift-service-ca.crt", "reconcileID": "7aa8b8fe-acca-46a4-ae78-fc7ee94f8049", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-a-694c02"} 2026-07-13T04:14:10Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-a-694c02"}, "namespace": "ai-tenant-e2e-shared-a-694c02", "name": "odh-trusted-ca-bundle", "reconcileID": "f1949ebf-e507-4314-9d0b-ea82e786897c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-694c02", "update": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:14:10Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-a-694c02"}, "namespace": "ai-tenant-e2e-shared-a-694c02", "name": "odh-trusted-ca-bundle", "reconcileID": "f1949ebf-e507-4314-9d0b-ea82e786897c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-694c02"} 2026-07-13T04:14:10Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-694c02"}, "namespace": "ai-tenant-e2e-shared-a-694c02", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6030be27-4dca-4425-9bca-5fbf6ccdc21e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-694c02"} 2026-07-13T04:14:10Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "559ea4b3-9139-4193-833b-f6512b3740f2", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress"} 2026-07-13T04:14:10Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "559ea4b3-9139-4193-833b-f6512b3740f2", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02-authn-ssl"} 2026-07-13T04:14:10Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "559ea4b3-9139-4193-833b-f6512b3740f2", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress"} 2026-07-13T04:14:10Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "8f1fe852-ea2a-4116-9ae4-de7098f34d52", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-13T04:14:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "3c50c121-813c-4ecd-9b05-ee56aad11d30", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-13T04:14:15Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "2607ec61-6074-43e8-992a-c543b81ae32c", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress"} 2026-07-13T04:14:15Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "2607ec61-6074-43e8-992a-c543b81ae32c", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7-authn-ssl"} 2026-07-13T04:14:15Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "2607ec61-6074-43e8-992a-c543b81ae32c", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress"} 2026-07-13T04:14:15Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "b06a7dc2-f471-4787-81bd-41eda950fa52", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress"} 2026-07-13T04:14:15Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "b06a7dc2-f471-4787-81bd-41eda950fa52", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7-authn-ssl"} 2026-07-13T04:14:15Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "b06a7dc2-f471-4787-81bd-41eda950fa52", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress"} 2026-07-13T04:14:15Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "c3e26da3-db7c-463d-a45a-e5c8382c4053", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress"} 2026-07-13T04:14:15Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "c3e26da3-db7c-463d-a45a-e5c8382c4053", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7-authn-ssl"} 2026-07-13T04:14:15Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "c3e26da3-db7c-463d-a45a-e5c8382c4053", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress"} 2026-07-13T04:14:15Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "081419f9-7e9e-40e0-ae1b-667f531c9cb3", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress"} 2026-07-13T04:14:15Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "081419f9-7e9e-40e0-ae1b-667f531c9cb3", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7-authn-ssl"} 2026-07-13T04:14:15Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "081419f9-7e9e-40e0-ae1b-667f531c9cb3", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress"} 2026-07-13T04:14:15Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "48bd69ae-f4e9-4244-84a2-bcddfec74eda", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress"} 2026-07-13T04:14:15Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "48bd69ae-f4e9-4244-84a2-bcddfec74eda", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7-authn-ssl"} 2026-07-13T04:14:15Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "48bd69ae-f4e9-4244-84a2-bcddfec74eda", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress"} 2026-07-13T04:14:15Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "56c120e3-e512-4d8c-8b0b-5b7a522764e6", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress"} 2026-07-13T04:14:15Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "56c120e3-e512-4d8c-8b0b-5b7a522764e6", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7-authn-ssl"} 2026-07-13T04:14:15Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "56c120e3-e512-4d8c-8b0b-5b7a522764e6", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress"} 2026-07-13T04:14:16Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "42f05220-3105-4909-94f6-e55897cefd4e", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress"} 2026-07-13T04:14:16Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "42f05220-3105-4909-94f6-e55897cefd4e", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7-authn-ssl"} 2026-07-13T04:14:16Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "42f05220-3105-4909-94f6-e55897cefd4e", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress"} 2026-07-13T04:14:18Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-b-830ad7"}, "namespace": "ai-tenant-e2e-shared-b-830ad7", "name": "openshift-service-ca.crt", "reconcileID": "b4b6671a-5703-44ce-a344-f72b57cfeb41", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-b-830ad7"} 2026-07-13T04:14:18Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-b-830ad7"}, "namespace": "ai-tenant-e2e-shared-b-830ad7", "name": "odh-trusted-ca-bundle", "reconcileID": "a8d4ee37-425e-4b62-9fd1-5c5f1deddc4d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-830ad7"} 2026-07-13T04:14:18Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-b-830ad7"}, "namespace": "ai-tenant-e2e-shared-b-830ad7", "name": "openshift-service-ca.crt", "reconcileID": "68b8a2e9-f1d4-447b-9ed2-fdf51396a4e5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-b-830ad7", "create": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:14:18Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-b-830ad7"}, "namespace": "ai-tenant-e2e-shared-b-830ad7", "name": "openshift-service-ca.crt", "reconcileID": "68b8a2e9-f1d4-447b-9ed2-fdf51396a4e5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-b-830ad7"} 2026-07-13T04:14:18Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-830ad7"}, "namespace": "ai-tenant-e2e-shared-b-830ad7", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8f91a29b-c4a0-4ff5-95bd-2d440162dd3d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-830ad7"} 2026-07-13T04:14:18Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-b-830ad7"}, "namespace": "ai-tenant-e2e-shared-b-830ad7", "name": "openshift-service-ca.crt", "reconcileID": "d7642137-ad17-4157-b5b4-bfd920f4d0a3", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-b-830ad7"} 2026-07-13T04:14:18Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-b-830ad7"}, "namespace": "ai-tenant-e2e-shared-b-830ad7", "name": "odh-trusted-ca-bundle", "reconcileID": "a53868a7-7dcc-4a29-9364-72926954e77d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-830ad7", "update": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:14:18Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-b-830ad7"}, "namespace": "ai-tenant-e2e-shared-b-830ad7", "name": "odh-trusted-ca-bundle", "reconcileID": "a53868a7-7dcc-4a29-9364-72926954e77d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-830ad7"} 2026-07-13T04:14:18Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-830ad7"}, "namespace": "ai-tenant-e2e-shared-b-830ad7", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1e1e1dda-e786-436f-9eb6-f84078dea852", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-830ad7"} 2026-07-13T04:14:18Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "60345b40-45d5-4fa2-81f8-9c1009eb9a98", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress"} 2026-07-13T04:14:18Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "60345b40-45d5-4fa2-81f8-9c1009eb9a98", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7-authn-ssl"} 2026-07-13T04:14:18Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "60345b40-45d5-4fa2-81f8-9c1009eb9a98", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress"} 2026-07-13T04:14:18Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "eb338875-74d4-42bc-b974-241792abd921", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress"} 2026-07-13T04:14:18Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "eb338875-74d4-42bc-b974-241792abd921", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02-authn-ssl"} 2026-07-13T04:14:18Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "eb338875-74d4-42bc-b974-241792abd921", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress"} 2026-07-13T04:14:18Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "21a6dde1-656f-48f1-b16c-5c2bc23069e8", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-13T04:14:25Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "3ee39cd8-a30e-4fe3-8e91-10546c1784e2", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress"} 2026-07-13T04:14:25Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "3ee39cd8-a30e-4fe3-8e91-10546c1784e2", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7-authn-ssl"} 2026-07-13T04:14:25Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "3ee39cd8-a30e-4fe3-8e91-10546c1784e2", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress"} 2026-07-13T04:14:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "26ac8a72-740c-4f3b-b4ad-4e34f4128c14", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-13T04:14:39Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "6d7df771-2539-42b6-9ddb-bf84b1601c7f", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-13T04:17:07Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-13T04:17:07Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-13T04:17:08Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-13T04:17:08Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-13T04:17:27Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "54ad2368-7e17-465e-ba2f-4b61c2154751", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-13T04:17:28Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "fb6db220-b8f6-43cc-a79a-a5e2cbd89153", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress"} 2026-07-13T04:17:28Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "fb6db220-b8f6-43cc-a79a-a5e2cbd89153", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02-authn-ssl"} 2026-07-13T04:17:28Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-694c02","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-694c02", "reconcileID": "fb6db220-b8f6-43cc-a79a-a5e2cbd89153", "Gateway": "e2e-shared-a-694c02", "namespace": "openshift-ingress"} 2026-07-13T04:17:33Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-694c02"}, "namespace": "ai-tenant-e2e-shared-a-694c02", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7685fbf9-956a-44ff-bc5c-57e4bc23cc55", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-694c02"} 2026-07-13T04:17:33Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-694c02"}, "namespace": "ai-tenant-e2e-shared-a-694c02", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7685fbf9-956a-44ff-bc5c-57e4bc23cc55", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-694c02", "create": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:17:33Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-694c02"}, "namespace": "ai-tenant-e2e-shared-a-694c02", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7685fbf9-956a-44ff-bc5c-57e4bc23cc55", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-694c02 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-13T04:17:33Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-a-694c02"}, "namespace": "ai-tenant-e2e-shared-a-694c02", "name": "odh-trusted-ca-bundle", "reconcileID": "9ba274d7-ab04-4b19-a26d-64ab3827238e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-694c02"} 2026-07-13T04:17:33Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-a-694c02"}, "namespace": "ai-tenant-e2e-shared-a-694c02", "name": "odh-trusted-ca-bundle", "reconcileID": "9ba274d7-ab04-4b19-a26d-64ab3827238e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-694c02"} 2026-07-13T04:17:33Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-694c02"}, "namespace": "ai-tenant-e2e-shared-a-694c02", "name": "openshift-service-ca.crt", "reconcileID": "0fcfeba9-394c-4013-a345-835ffe223433", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-a-694c02"} 2026-07-13T04:17:33Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-694c02"}, "namespace": "ai-tenant-e2e-shared-a-694c02", "name": "openshift-service-ca.crt", "reconcileID": "0fcfeba9-394c-4013-a345-835ffe223433", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-a-694c02"} 2026-07-13T04:17:33Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-694c02"}, "namespace": "ai-tenant-e2e-shared-a-694c02", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f1a98bed-3a17-4af0-940a-456061baa994", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-694c02"} 2026-07-13T04:17:33Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-694c02"}, "namespace": "ai-tenant-e2e-shared-a-694c02", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f1a98bed-3a17-4af0-940a-456061baa994", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-694c02"} 2026-07-13T04:17:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "70b50a90-d1c9-426f-bd68-987a937f45ad", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-13T04:17:35Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "cc8b0958-998f-41e0-b103-7613c97f0b72", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress"} 2026-07-13T04:17:35Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "cc8b0958-998f-41e0-b103-7613c97f0b72", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7-authn-ssl"} 2026-07-13T04:17:35Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-830ad7","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-830ad7", "reconcileID": "cc8b0958-998f-41e0-b103-7613c97f0b72", "Gateway": "e2e-shared-b-830ad7", "namespace": "openshift-ingress"} 2026-07-13T04:17:40Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-830ad7"}, "namespace": "ai-tenant-e2e-shared-b-830ad7", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9e484477-f0c2-4673-9214-1538260b5fd0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-830ad7"} 2026-07-13T04:17:40Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-830ad7"}, "namespace": "ai-tenant-e2e-shared-b-830ad7", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9e484477-f0c2-4673-9214-1538260b5fd0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-830ad7", "create": "odh-kserve-custom-ca-bundle"} 2026-07-13T04:17:40Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-830ad7"}, "namespace": "ai-tenant-e2e-shared-b-830ad7", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9e484477-f0c2-4673-9214-1538260b5fd0", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-830ad7 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-13T04:17:40Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-b-830ad7"}, "namespace": "ai-tenant-e2e-shared-b-830ad7", "name": "odh-trusted-ca-bundle", "reconcileID": "3fac6fa6-9466-4091-a85c-67fee57607d5", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-830ad7"} 2026-07-13T04:17:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-b-830ad7"}, "namespace": "ai-tenant-e2e-shared-b-830ad7", "name": "odh-trusted-ca-bundle", "reconcileID": "3fac6fa6-9466-4091-a85c-67fee57607d5", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-830ad7"} 2026-07-13T04:17:40Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-b-830ad7"}, "namespace": "ai-tenant-e2e-shared-b-830ad7", "name": "openshift-service-ca.crt", "reconcileID": "816de61a-7904-4e6c-8def-59f4f27f9197", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-b-830ad7"} 2026-07-13T04:17:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-b-830ad7"}, "namespace": "ai-tenant-e2e-shared-b-830ad7", "name": "openshift-service-ca.crt", "reconcileID": "816de61a-7904-4e6c-8def-59f4f27f9197", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-b-830ad7"} 2026-07-13T04:17:40Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-830ad7"}, "namespace": "ai-tenant-e2e-shared-b-830ad7", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c51426b8-662a-4eb4-ae42-be32b134e9b5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-830ad7"} 2026-07-13T04:17:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-830ad7"}, "namespace": "ai-tenant-e2e-shared-b-830ad7", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c51426b8-662a-4eb4-ae42-be32b134e9b5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-830ad7"}