2026-07-08T20:31:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-3ee4b0cc"}, "namespace": "ai-tenant-e2e-mt-3ee4b0cc", "name": "openshift-service-ca.crt", "reconcileID": "151ae1b9-3b98-499e-af1f-3f5c458d9238", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-3ee4b0cc"} 2026-07-08T20:31:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-3ee4b0cc"}, "namespace": "ai-tenant-e2e-mt-3ee4b0cc", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1fc444ea-a42d-4e5b-a69e-01661985e81c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-3ee4b0cc"} 2026-07-08T20:31:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-3ee4b0cc"}, "namespace": "ai-tenant-e2e-mt-3ee4b0cc", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1fc444ea-a42d-4e5b-a69e-01661985e81c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-3ee4b0cc"} 2026-07-08T20:31:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-69ee637e"}, "namespace": "ai-tenant-e2e-mt-69ee637e", "name": "openshift-service-ca.crt", "reconcileID": "d84395ff-6be3-492c-94e2-73b731ea0006", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-69ee637e"} 2026-07-08T20:31:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-69ee637e"}, "namespace": "ai-tenant-e2e-mt-69ee637e", "name": "odh-trusted-ca-bundle", "reconcileID": "808f106d-415b-4303-84ca-c0b15883d950", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-69ee637e"} 2026-07-08T20:31:55Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-69ee637e"}, "namespace": "ai-tenant-e2e-mt-69ee637e", "name": "openshift-service-ca.crt", "reconcileID": "cfaae615-6d60-4620-be3f-0d66c890fc2a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-69ee637e", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T20:31:55Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-69ee637e"}, "namespace": "ai-tenant-e2e-mt-69ee637e", "name": "openshift-service-ca.crt", "reconcileID": "cfaae615-6d60-4620-be3f-0d66c890fc2a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-69ee637e"} 2026-07-08T20:31:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-69ee637e"}, "namespace": "ai-tenant-e2e-mt-69ee637e", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "be7880de-f730-4355-87a6-7e71cac68c96", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-69ee637e"} 2026-07-08T20:31:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-69ee637e"}, "namespace": "ai-tenant-e2e-mt-69ee637e", "name": "openshift-service-ca.crt", "reconcileID": "afebbef5-0c75-42c0-b5a3-f8ef8b51708b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-69ee637e"} 2026-07-08T20:31:55Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-69ee637e"}, "namespace": "ai-tenant-e2e-mt-69ee637e", "name": "odh-trusted-ca-bundle", "reconcileID": "9f4cd47a-61c5-4128-8c05-881bc65b237d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-69ee637e", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T20:31:55Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-69ee637e"}, "namespace": "ai-tenant-e2e-mt-69ee637e", "name": "odh-trusted-ca-bundle", "reconcileID": "9f4cd47a-61c5-4128-8c05-881bc65b237d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-69ee637e"} 2026-07-08T20:31:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-69ee637e"}, "namespace": "ai-tenant-e2e-mt-69ee637e", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6a90ffca-4bf8-49d7-8e9c-213e9adc9b02", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-69ee637e"} 2026-07-08T20:32:33Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-69ee637e"}, "namespace": "ai-tenant-e2e-mt-69ee637e", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b558aa89-55e7-4df2-996c-f8b1f8723aa8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-69ee637e"} 2026-07-08T20:32:33Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-69ee637e"}, "namespace": "ai-tenant-e2e-mt-69ee637e", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b558aa89-55e7-4df2-996c-f8b1f8723aa8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-69ee637e", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T20:32:33Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-69ee637e"}, "namespace": "ai-tenant-e2e-mt-69ee637e", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b558aa89-55e7-4df2-996c-f8b1f8723aa8", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-69ee637e because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T20:32:33Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-69ee637e"}, "namespace": "ai-tenant-e2e-mt-69ee637e", "name": "odh-trusted-ca-bundle", "reconcileID": "b577175a-2a94-4cfb-988a-3849fbad5f8a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-69ee637e"} 2026-07-08T20:32:33Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-69ee637e"}, "namespace": "ai-tenant-e2e-mt-69ee637e", "name": "odh-trusted-ca-bundle", "reconcileID": "b577175a-2a94-4cfb-988a-3849fbad5f8a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-69ee637e"} 2026-07-08T20:32:33Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-69ee637e"}, "namespace": "ai-tenant-e2e-mt-69ee637e", "name": "openshift-service-ca.crt", "reconcileID": "02829b8d-49c2-47a3-bb4d-bdb467da71de", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-69ee637e"} 2026-07-08T20:32:33Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-69ee637e"}, "namespace": "ai-tenant-e2e-mt-69ee637e", "name": "openshift-service-ca.crt", "reconcileID": "02829b8d-49c2-47a3-bb4d-bdb467da71de", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-69ee637e"} 2026-07-08T20:32:33Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-69ee637e"}, "namespace": "ai-tenant-e2e-mt-69ee637e", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bb0c524a-efab-466c-8ebc-d6c3054fa0bf", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-69ee637e"} 2026-07-08T20:32:33Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-69ee637e"}, "namespace": "ai-tenant-e2e-mt-69ee637e", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bb0c524a-efab-466c-8ebc-d6c3054fa0bf", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-69ee637e"} 2026-07-08T20:32:34Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "e6e2e375-4abd-4421-b583-a55456c63fa8", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:32:34Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "e6e2e375-4abd-4421-b583-a55456c63fa8", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress", "name": "e2e-inf-a-930853-authn-ssl"} 2026-07-08T20:32:34Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "e6e2e375-4abd-4421-b583-a55456c63fa8", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:32:34Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "2a568b4c-40ad-4642-ad82-0151f2ec0277", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:32:34Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "2a568b4c-40ad-4642-ad82-0151f2ec0277", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress", "name": "e2e-inf-a-930853-authn-ssl"} 2026-07-08T20:32:34Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "2a568b4c-40ad-4642-ad82-0151f2ec0277", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:32:34Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "b9e75a16-b925-47fa-8de5-5c031afb2a42", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:32:34Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "b9e75a16-b925-47fa-8de5-5c031afb2a42", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress", "name": "e2e-inf-a-930853-authn-ssl"} 2026-07-08T20:32:34Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "b9e75a16-b925-47fa-8de5-5c031afb2a42", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:32:34Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "77fab658-6c3b-49ae-b97d-5bd3a8ddac86", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:32:34Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "77fab658-6c3b-49ae-b97d-5bd3a8ddac86", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress", "name": "e2e-inf-a-930853-authn-ssl"} 2026-07-08T20:32:34Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "77fab658-6c3b-49ae-b97d-5bd3a8ddac86", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:32:34Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "8565ee9d-ac12-4bd0-aac6-cc1ba744bdf3", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:32:34Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "8565ee9d-ac12-4bd0-aac6-cc1ba744bdf3", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress", "name": "e2e-inf-a-930853-authn-ssl"} 2026-07-08T20:32:34Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "8565ee9d-ac12-4bd0-aac6-cc1ba744bdf3", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:32:34Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "8355ef89-2bc0-4c5a-b328-2564a10f9dce", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:32:34Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "8355ef89-2bc0-4c5a-b328-2564a10f9dce", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress", "name": "e2e-inf-a-930853-authn-ssl"} 2026-07-08T20:32:34Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "8355ef89-2bc0-4c5a-b328-2564a10f9dce", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:32:35Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "2951df75-061c-4b53-b92d-71f9dffd648e", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:32:35Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "2951df75-061c-4b53-b92d-71f9dffd648e", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress", "name": "e2e-inf-a-930853-authn-ssl"} 2026-07-08T20:32:35Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "2951df75-061c-4b53-b92d-71f9dffd648e", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:32:37Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "openshift-service-ca.crt", "reconcileID": "49c07651-6931-4801-b030-ab76d4488c31", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:32:37Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "odh-trusted-ca-bundle", "reconcileID": "0affb3e1-e21c-4ce6-8832-824323d2d719", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:32:37Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "openshift-service-ca.crt", "reconcileID": "ba4fce57-33ba-475e-a5c6-6a56deeca8e1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-930853", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T20:32:37Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "openshift-service-ca.crt", "reconcileID": "ba4fce57-33ba-475e-a5c6-6a56deeca8e1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:32:37Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "openshift-service-ca.crt", "reconcileID": "636326ba-cf0d-4510-887b-16c663b78633", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-930853", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T20:32:37Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "openshift-service-ca.crt", "reconcileID": "636326ba-cf0d-4510-887b-16c663b78633", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T20:32:37Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8a43398a-3abc-46d6-a86e-2965d9d46afe", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:32:37Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "openshift-service-ca.crt", "reconcileID": "ae9d1452-1b4a-4a0a-a673-7033bef0ab0b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:32:37Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "odh-trusted-ca-bundle", "reconcileID": "8b876ab7-4989-418d-80d4-29fada604cc5", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-930853", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T20:32:37Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "odh-trusted-ca-bundle", "reconcileID": "8b876ab7-4989-418d-80d4-29fada604cc5", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:32:37Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a6854018-d117-4542-b7c0-5f71c30a5ec7", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:32:37Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "9c8a4738-19a0-4ff2-9422-5f799b46c9c4", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:32:37Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "9c8a4738-19a0-4ff2-9422-5f799b46c9c4", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress", "name": "e2e-inf-a-930853-authn-ssl"} 2026-07-08T20:32:37Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "9c8a4738-19a0-4ff2-9422-5f799b46c9c4", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:32:37Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "468c7e07-faea-4e1d-86ab-8f66dc89f73e", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T20:32:42Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "144025de-e4e2-4fe0-9272-017efa711b8c", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:32:42Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "144025de-e4e2-4fe0-9272-017efa711b8c", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress", "name": "e2e-inf-b-879445-authn-ssl"} 2026-07-08T20:32:42Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "144025de-e4e2-4fe0-9272-017efa711b8c", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:32:42Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "715f9470-817f-4f99-b007-fcdd81f8930b", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:32:42Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "715f9470-817f-4f99-b007-fcdd81f8930b", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress", "name": "e2e-inf-b-879445-authn-ssl"} 2026-07-08T20:32:42Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "715f9470-817f-4f99-b007-fcdd81f8930b", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:32:42Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "3ebbfe4c-f611-4b40-8a9c-5d17a1a9e27b", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:32:42Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "3ebbfe4c-f611-4b40-8a9c-5d17a1a9e27b", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress", "name": "e2e-inf-b-879445-authn-ssl"} 2026-07-08T20:32:42Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "3ebbfe4c-f611-4b40-8a9c-5d17a1a9e27b", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:32:42Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "52a6f32a-9db1-463b-8348-51fb5154144a", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:32:42Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "52a6f32a-9db1-463b-8348-51fb5154144a", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress", "name": "e2e-inf-b-879445-authn-ssl"} 2026-07-08T20:32:42Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "52a6f32a-9db1-463b-8348-51fb5154144a", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:32:42Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "1ea0fff6-7271-45e3-8db8-788aa57b743e", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:32:42Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "1ea0fff6-7271-45e3-8db8-788aa57b743e", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress", "name": "e2e-inf-a-930853-authn-ssl"} 2026-07-08T20:32:42Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "1ea0fff6-7271-45e3-8db8-788aa57b743e", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:32:42Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "a98c0792-7541-4c63-adcf-133b4d43fc4f", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:32:42Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "a98c0792-7541-4c63-adcf-133b4d43fc4f", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress", "name": "e2e-inf-b-879445-authn-ssl"} 2026-07-08T20:32:42Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "a98c0792-7541-4c63-adcf-133b4d43fc4f", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:32:42Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "ac8f7476-5e2f-4a3f-a15c-a9cdbc33a75a", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:32:42Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "ac8f7476-5e2f-4a3f-a15c-a9cdbc33a75a", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress", "name": "e2e-inf-b-879445-authn-ssl"} 2026-07-08T20:32:42Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "ac8f7476-5e2f-4a3f-a15c-a9cdbc33a75a", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:32:43Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "4d006ed7-d54d-4471-af5e-225411defcf1", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:32:43Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "4d006ed7-d54d-4471-af5e-225411defcf1", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress", "name": "e2e-inf-b-879445-authn-ssl"} 2026-07-08T20:32:43Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "4d006ed7-d54d-4471-af5e-225411defcf1", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:32:45Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "openshift-service-ca.crt", "reconcileID": "7b005bcc-828e-467d-8e4d-cf8a5e4e244e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:32:45Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "odh-trusted-ca-bundle", "reconcileID": "2cb6d256-ba86-45c7-8f27-2e0e01dd721c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:32:45Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "openshift-service-ca.crt", "reconcileID": "36e3b9db-8dc5-4a1d-b474-59d3af1c31bb", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-879445", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T20:32:45Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "openshift-service-ca.crt", "reconcileID": "36e3b9db-8dc5-4a1d-b474-59d3af1c31bb", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:32:45Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "openshift-service-ca.crt", "reconcileID": "6c07c666-3310-41ea-ab1e-2e0aab81b9c1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:32:45Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "799ce463-a328-4e99-a683-81749b6d7b9f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:32:45Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "odh-trusted-ca-bundle", "reconcileID": "3f0aec26-6e4b-4512-a8f5-9c1dc9e9e9e6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-879445", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T20:32:45Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "odh-trusted-ca-bundle", "reconcileID": "3f0aec26-6e4b-4512-a8f5-9c1dc9e9e9e6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:32:45Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a941b762-58ea-412c-845e-6c1fd8efb71b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:32:45Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "90c8f7fe-4afd-4d29-82a8-2b36644d9d57", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:32:45Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "90c8f7fe-4afd-4d29-82a8-2b36644d9d57", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress", "name": "e2e-inf-b-879445-authn-ssl"} 2026-07-08T20:32:45Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "90c8f7fe-4afd-4d29-82a8-2b36644d9d57", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:32:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "a04f0dba-6f83-4f1f-b0c5-1786faf4ff94", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T20:32:50Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "aa6b0515-0be9-4439-a236-3e35f9d0e564", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:32:50Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "aa6b0515-0be9-4439-a236-3e35f9d0e564", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:32:50Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "aa6b0515-0be9-4439-a236-3e35f9d0e564", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:32:50Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "0d10e920-bef4-41ad-bc0d-269e740ec586", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:32:50Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "aa6b0515-0be9-4439-a236-3e35f9d0e564", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853-kserve-route-authn"} 2026-07-08T20:32:50Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "aa6b0515-0be9-4439-a236-3e35f9d0e564", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853", "authType": "userdefined"} 2026-07-08T20:32:50Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "aa6b0515-0be9-4439-a236-3e35f9d0e564", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:32:50Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "0d10e920-bef4-41ad-bc0d-269e740ec586", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress", "name": "e2e-inf-a-930853-authn-ssl"} 2026-07-08T20:32:50Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "0d10e920-bef4-41ad-bc0d-269e740ec586", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:32:50Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "f119953f-d853-4e01-9a38-a8f0bc6d85dc", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:32:50Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "f119953f-d853-4e01-9a38-a8f0bc6d85dc", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:32:50Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "f119953f-d853-4e01-9a38-a8f0bc6d85dc", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:32:50Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "f119953f-d853-4e01-9a38-a8f0bc6d85dc", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853-kserve-route-authn"} 2026-07-08T20:32:50Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "f119953f-d853-4e01-9a38-a8f0bc6d85dc", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853", "authType": "userdefined"} 2026-07-08T20:32:50Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "f119953f-d853-4e01-9a38-a8f0bc6d85dc", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:32:51Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "8ae32533-ba9a-4ec0-9088-f31eb1089639", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:32:51Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "8ae32533-ba9a-4ec0-9088-f31eb1089639", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress", "name": "e2e-inf-b-879445-authn-ssl"} 2026-07-08T20:32:51Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "8ae32533-ba9a-4ec0-9088-f31eb1089639", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:32:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "2191112c-1c80-46b0-ad6c-a406489701fa", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T20:33:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "a8b5bbd6-b1f0-42ee-bbae-bdc6766df881", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T20:33:11Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "201d82e9-b28f-431f-b213-a2251b3af3fa", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:33:11Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "201d82e9-b28f-431f-b213-a2251b3af3fa", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress", "name": "e2e-inf-a-930853-authn-ssl"} 2026-07-08T20:33:11Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "201d82e9-b28f-431f-b213-a2251b3af3fa", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:33:12Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "3dfc829c-e2eb-409f-b48a-02f81d90c70c", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:33:12Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "3dfc829c-e2eb-409f-b48a-02f81d90c70c", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:33:12Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "3dfc829c-e2eb-409f-b48a-02f81d90c70c", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:33:12Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "3dfc829c-e2eb-409f-b48a-02f81d90c70c", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853-kserve-route-authn"} 2026-07-08T20:33:12Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "3dfc829c-e2eb-409f-b48a-02f81d90c70c", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853", "authType": "userdefined"} 2026-07-08T20:33:12Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "3dfc829c-e2eb-409f-b48a-02f81d90c70c", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:33:22Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "d648c0a5-83ea-47a5-8e81-2bfaee0b40a4", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:33:22Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "d648c0a5-83ea-47a5-8e81-2bfaee0b40a4", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress", "name": "e2e-inf-a-930853-authn-ssl"} 2026-07-08T20:33:22Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "d648c0a5-83ea-47a5-8e81-2bfaee0b40a4", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:33:37Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "b17a602a-0e64-4a58-9ba6-5a58ae29c8a3", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:33:37Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "b17a602a-0e64-4a58-9ba6-5a58ae29c8a3", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:33:37Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "b17a602a-0e64-4a58-9ba6-5a58ae29c8a3", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:33:37Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "b17a602a-0e64-4a58-9ba6-5a58ae29c8a3", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853-kserve-route-authn"} 2026-07-08T20:33:37Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "b17a602a-0e64-4a58-9ba6-5a58ae29c8a3", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853", "authType": "userdefined"} 2026-07-08T20:33:37Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "b17a602a-0e64-4a58-9ba6-5a58ae29c8a3", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:33:42Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "f64a2462-4f91-49dc-8879-ae25c9f939d2", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:33:42Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "f64a2462-4f91-49dc-8879-ae25c9f939d2", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:33:42Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "f64a2462-4f91-49dc-8879-ae25c9f939d2", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:33:42Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "f589b9b3-f506-4b89-94f0-e0565396ce71", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:33:42Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "f64a2462-4f91-49dc-8879-ae25c9f939d2", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445-kserve-route-authn"} 2026-07-08T20:33:42Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "f64a2462-4f91-49dc-8879-ae25c9f939d2", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445", "authType": "userdefined"} 2026-07-08T20:33:42Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "f64a2462-4f91-49dc-8879-ae25c9f939d2", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:33:42Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "f589b9b3-f506-4b89-94f0-e0565396ce71", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress", "name": "e2e-inf-b-879445-authn-ssl"} 2026-07-08T20:33:42Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "f589b9b3-f506-4b89-94f0-e0565396ce71", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:33:42Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "0e21ad4f-e7cc-482f-9a45-cb6d6c438f59", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:33:42Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "0e21ad4f-e7cc-482f-9a45-cb6d6c438f59", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:33:42Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "0e21ad4f-e7cc-482f-9a45-cb6d6c438f59", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:33:42Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "0e21ad4f-e7cc-482f-9a45-cb6d6c438f59", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445-kserve-route-authn"} 2026-07-08T20:33:42Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "0e21ad4f-e7cc-482f-9a45-cb6d6c438f59", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445", "authType": "userdefined"} 2026-07-08T20:33:42Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "0e21ad4f-e7cc-482f-9a45-cb6d6c438f59", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:33:45Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "3ec93fc1-4ddb-4cb5-b051-db5a225b2bb1", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:33:45Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "3ec93fc1-4ddb-4cb5-b051-db5a225b2bb1", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress", "name": "e2e-inf-b-879445-authn-ssl"} 2026-07-08T20:33:45Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "3ec93fc1-4ddb-4cb5-b051-db5a225b2bb1", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:33:46Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "ce725eb3-22c2-47cf-9196-50e30680f140", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:33:46Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "ce725eb3-22c2-47cf-9196-50e30680f140", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:33:46Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "ce725eb3-22c2-47cf-9196-50e30680f140", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:33:46Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "ce725eb3-22c2-47cf-9196-50e30680f140", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445-kserve-route-authn"} 2026-07-08T20:33:46Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "ce725eb3-22c2-47cf-9196-50e30680f140", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445", "authType": "userdefined"} 2026-07-08T20:33:46Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "ce725eb3-22c2-47cf-9196-50e30680f140", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:33:59Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "70a98cf9-847f-49dd-85ed-16d92daf4d73", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:33:59Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "70a98cf9-847f-49dd-85ed-16d92daf4d73", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress", "name": "e2e-inf-b-879445-authn-ssl"} 2026-07-08T20:33:59Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "70a98cf9-847f-49dd-85ed-16d92daf4d73", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:34:16Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "181de2c9-e168-4ae8-94a2-a34484cf3cf6", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:34:16Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "181de2c9-e168-4ae8-94a2-a34484cf3cf6", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:34:16Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "181de2c9-e168-4ae8-94a2-a34484cf3cf6", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:34:16Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "181de2c9-e168-4ae8-94a2-a34484cf3cf6", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445-kserve-route-authn"} 2026-07-08T20:34:16Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "181de2c9-e168-4ae8-94a2-a34484cf3cf6", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445", "authType": "userdefined"} 2026-07-08T20:34:16Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "181de2c9-e168-4ae8-94a2-a34484cf3cf6", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:34:24Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "a0163437-331b-4ee1-a17c-df8629579bc7", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:34:24Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "a0163437-331b-4ee1-a17c-df8629579bc7", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853"} 2026-07-08T20:34:24Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "a0163437-331b-4ee1-a17c-df8629579bc7", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:34:24Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "a0163437-331b-4ee1-a17c-df8629579bc7", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:34:24Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "a0163437-331b-4ee1-a17c-df8629579bc7", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853-kserve-route-authn"} 2026-07-08T20:34:24Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "a0163437-331b-4ee1-a17c-df8629579bc7", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853-kserve-route-authn"} 2026-07-08T20:34:24Z DEBUG Triggering LLMInferenceService Cleanup for Namespace {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "a0163437-331b-4ee1-a17c-df8629579bc7", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:34:24Z DEBUG Cleaning up LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "a0163437-331b-4ee1-a17c-df8629579bc7", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:34:24Z INFO Stop LLMInferenceService reconciliation {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-930853","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "test-model-930853", "reconcileID": "85cfbe37-01d3-4d56-a179-2cce910fca9f", "LLMInferenceService": "test-model-930853", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:34:24Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "73172052-21ef-4a7b-a946-b8a9f1f3221f", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:34:24Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "73172052-21ef-4a7b-a946-b8a9f1f3221f", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress", "name": "e2e-inf-a-930853-authn-ssl"} 2026-07-08T20:34:24Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "73172052-21ef-4a7b-a946-b8a9f1f3221f", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:34:24Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "5041a23e-80f2-4f1e-bd0d-fe0bd7bb3f12", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:34:24Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "5041a23e-80f2-4f1e-bd0d-fe0bd7bb3f12", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress", "name": "e2e-inf-a-930853-authn-ssl"} 2026-07-08T20:34:24Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "5041a23e-80f2-4f1e-bd0d-fe0bd7bb3f12", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:34:25Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "5c7250bf-39fa-4a16-b2db-7f9a2e50dbab", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:34:25Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "5c7250bf-39fa-4a16-b2db-7f9a2e50dbab", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445"} 2026-07-08T20:34:25Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "5c7250bf-39fa-4a16-b2db-7f9a2e50dbab", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:34:25Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "5c7250bf-39fa-4a16-b2db-7f9a2e50dbab", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:34:25Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "5c7250bf-39fa-4a16-b2db-7f9a2e50dbab", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445-kserve-route-authn"} 2026-07-08T20:34:25Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "5c7250bf-39fa-4a16-b2db-7f9a2e50dbab", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445-kserve-route-authn"} 2026-07-08T20:34:25Z DEBUG Triggering LLMInferenceService Cleanup for Namespace {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "5c7250bf-39fa-4a16-b2db-7f9a2e50dbab", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:34:25Z DEBUG Cleaning up LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "5c7250bf-39fa-4a16-b2db-7f9a2e50dbab", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:34:25Z INFO Stop LLMInferenceService reconciliation {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-879445","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "test-model-879445", "reconcileID": "e552e047-0d8a-477a-a294-19a9cfa4b874", "LLMInferenceService": "test-model-879445", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:34:25Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "c6745d15-9b8f-4f3f-830b-e5c79a30e88e", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:34:25Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "c6745d15-9b8f-4f3f-830b-e5c79a30e88e", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress", "name": "e2e-inf-b-879445-authn-ssl"} 2026-07-08T20:34:25Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "c6745d15-9b8f-4f3f-830b-e5c79a30e88e", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:34:25Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "0c101415-eeb3-4ab7-81b3-761c9db1697e", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:34:25Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "0c101415-eeb3-4ab7-81b3-761c9db1697e", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress", "name": "e2e-inf-b-879445-authn-ssl"} 2026-07-08T20:34:25Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "0c101415-eeb3-4ab7-81b3-761c9db1697e", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:34:26Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "c994154d-ed42-4f9b-8b52-9236aa2d3f4b", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T20:34:27Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "9d507da5-5c1c-4fe4-a69d-d23a9e6e31e6", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:34:27Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "9d507da5-5c1c-4fe4-a69d-d23a9e6e31e6", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress", "name": "e2e-inf-a-930853-authn-ssl"} 2026-07-08T20:34:27Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "9d507da5-5c1c-4fe4-a69d-d23a9e6e31e6", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:35:08Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d5f52b4f-e16a-4d4e-b7ce-1406616dc6f9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:35:08Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d5f52b4f-e16a-4d4e-b7ce-1406616dc6f9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-930853", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T20:35:08Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d5f52b4f-e16a-4d4e-b7ce-1406616dc6f9", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-inf-a-930853 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T20:35:08Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "odh-trusted-ca-bundle", "reconcileID": "7964b100-86e1-423a-85fb-62bebf6ecd2d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:35:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "odh-trusted-ca-bundle", "reconcileID": "7964b100-86e1-423a-85fb-62bebf6ecd2d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:35:08Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "openshift-service-ca.crt", "reconcileID": "02214fc1-0b30-41f5-9695-55cea90614b7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:35:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "openshift-service-ca.crt", "reconcileID": "02214fc1-0b30-41f5-9695-55cea90614b7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:35:08Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5eac271e-0375-4b54-a0db-ea10f98ccb5d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:35:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-930853"}, "namespace": "ai-tenant-e2e-inf-a-930853", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5eac271e-0375-4b54-a0db-ea10f98ccb5d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-930853"} 2026-07-08T20:35:08Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-930853","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-930853", "reconcileID": "e4308d17-dda2-413d-ab04-4f930d045800", "Gateway": "e2e-inf-a-930853", "namespace": "openshift-ingress"} 2026-07-08T20:35:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "ac251e7d-8ebb-4a71-bb95-ab44e84cfae4", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T20:35:10Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "e3a62b8e-569b-4090-a749-564a19b0028a", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:35:10Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "e3a62b8e-569b-4090-a749-564a19b0028a", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress", "name": "e2e-inf-b-879445-authn-ssl"} 2026-07-08T20:35:10Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "e3a62b8e-569b-4090-a749-564a19b0028a", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:35:14Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "2596b2b0-6e0c-4a3d-aef7-5b2ca4642647", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T20:35:15Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1afc79b7-9842-4a1c-8ea2-13503db0bd5c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:35:15Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1afc79b7-9842-4a1c-8ea2-13503db0bd5c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-879445", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T20:35:15Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1afc79b7-9842-4a1c-8ea2-13503db0bd5c", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-inf-b-879445 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T20:35:15Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "odh-trusted-ca-bundle", "reconcileID": "fa2c8837-1fb5-41a2-a21f-43e5ab94a00b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:35:15Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "odh-trusted-ca-bundle", "reconcileID": "fa2c8837-1fb5-41a2-a21f-43e5ab94a00b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:35:15Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "openshift-service-ca.crt", "reconcileID": "4657783d-e4f6-4c83-a23e-e741ab71cf18", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:35:15Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "openshift-service-ca.crt", "reconcileID": "4657783d-e4f6-4c83-a23e-e741ab71cf18", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:35:15Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7bb92619-492e-4495-b1c7-d61867e3fd73", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:35:15Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-879445"}, "namespace": "ai-tenant-e2e-inf-b-879445", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7bb92619-492e-4495-b1c7-d61867e3fd73", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-879445"} 2026-07-08T20:35:16Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-879445","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-879445", "reconcileID": "9b7eb024-3c43-4928-9a58-31bd3020ae77", "Gateway": "e2e-inf-b-879445", "namespace": "openshift-ingress"} 2026-07-08T20:35:17Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "f89c4be5-0bbc-46f1-b1f2-c5f7b2b7ab75", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress"} 2026-07-08T20:35:17Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "f89c4be5-0bbc-46f1-b1f2-c5f7b2b7ab75", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress", "name": "e2e-api-a-575204-authn-ssl"} 2026-07-08T20:35:17Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "f89c4be5-0bbc-46f1-b1f2-c5f7b2b7ab75", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress"} 2026-07-08T20:35:17Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "04d3a66b-ae28-4ba5-8a31-36769c77fab5", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress"} 2026-07-08T20:35:17Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "04d3a66b-ae28-4ba5-8a31-36769c77fab5", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress", "name": "e2e-api-a-575204-authn-ssl"} 2026-07-08T20:35:17Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "04d3a66b-ae28-4ba5-8a31-36769c77fab5", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress"} 2026-07-08T20:35:17Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "4289c2a1-4905-4a59-bebc-144bb2a8140a", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress"} 2026-07-08T20:35:17Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "4289c2a1-4905-4a59-bebc-144bb2a8140a", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress", "name": "e2e-api-a-575204-authn-ssl"} 2026-07-08T20:35:17Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "4289c2a1-4905-4a59-bebc-144bb2a8140a", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress"} 2026-07-08T20:35:17Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "52eaf2b2-3686-4b21-8484-f4c355e2ef6b", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress"} 2026-07-08T20:35:17Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "52eaf2b2-3686-4b21-8484-f4c355e2ef6b", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress", "name": "e2e-api-a-575204-authn-ssl"} 2026-07-08T20:35:17Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "52eaf2b2-3686-4b21-8484-f4c355e2ef6b", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress"} 2026-07-08T20:35:17Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "d02f0394-cebf-4209-8395-e3d24768e35d", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress"} 2026-07-08T20:35:17Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "d02f0394-cebf-4209-8395-e3d24768e35d", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress", "name": "e2e-api-a-575204-authn-ssl"} 2026-07-08T20:35:17Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "d02f0394-cebf-4209-8395-e3d24768e35d", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress"} 2026-07-08T20:35:17Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "866c0629-5393-4fe1-8397-8a0826b6f6dc", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress"} 2026-07-08T20:35:17Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "866c0629-5393-4fe1-8397-8a0826b6f6dc", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress", "name": "e2e-api-a-575204-authn-ssl"} 2026-07-08T20:35:17Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "866c0629-5393-4fe1-8397-8a0826b6f6dc", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress"} 2026-07-08T20:35:19Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "7d5fccfc-ccfa-497b-b3e3-c6d720121870", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress"} 2026-07-08T20:35:19Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "7d5fccfc-ccfa-497b-b3e3-c6d720121870", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress", "name": "e2e-api-a-575204-authn-ssl"} 2026-07-08T20:35:19Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "7d5fccfc-ccfa-497b-b3e3-c6d720121870", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress"} 2026-07-08T20:35:21Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-575204"}, "namespace": "ai-tenant-e2e-api-a-575204", "name": "openshift-service-ca.crt", "reconcileID": "90aac8ef-2955-41ca-aaf3-9049fa56a1da", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-575204"} 2026-07-08T20:35:21Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-575204"}, "namespace": "ai-tenant-e2e-api-a-575204", "name": "odh-trusted-ca-bundle", "reconcileID": "bb8b5d41-fc97-4d1a-b354-96c6a750f1b7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-575204"} 2026-07-08T20:35:21Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-575204"}, "namespace": "ai-tenant-e2e-api-a-575204", "name": "openshift-service-ca.crt", "reconcileID": "db1e0fef-30d6-4649-a9a1-300be5f48525", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-575204", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T20:35:21Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-575204"}, "namespace": "ai-tenant-e2e-api-a-575204", "name": "openshift-service-ca.crt", "reconcileID": "db1e0fef-30d6-4649-a9a1-300be5f48525", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-575204"} 2026-07-08T20:35:21Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-575204"}, "namespace": "ai-tenant-e2e-api-a-575204", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bb30e352-d7ee-4857-84b0-795e2a9f192e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-575204"} 2026-07-08T20:35:21Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-575204"}, "namespace": "ai-tenant-e2e-api-a-575204", "name": "openshift-service-ca.crt", "reconcileID": "95bbbdba-3d53-43fd-a6b2-026d69362bc1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-575204"} 2026-07-08T20:35:21Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-575204"}, "namespace": "ai-tenant-e2e-api-a-575204", "name": "odh-trusted-ca-bundle", "reconcileID": "e1762ba6-fc80-4ba2-a87f-26386291b224", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-575204", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T20:35:21Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-575204"}, "namespace": "ai-tenant-e2e-api-a-575204", "name": "odh-trusted-ca-bundle", "reconcileID": "e1762ba6-fc80-4ba2-a87f-26386291b224", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-575204"} 2026-07-08T20:35:21Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-575204"}, "namespace": "ai-tenant-e2e-api-a-575204", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d6b90ec8-4698-4a5a-a496-c20c95edb0eb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-575204"} 2026-07-08T20:35:21Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "f283e150-0da6-46fe-8d33-4a7e62a71407", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress"} 2026-07-08T20:35:21Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "f283e150-0da6-46fe-8d33-4a7e62a71407", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress", "name": "e2e-api-a-575204-authn-ssl"} 2026-07-08T20:35:21Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "f283e150-0da6-46fe-8d33-4a7e62a71407", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress"} 2026-07-08T20:35:21Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "23f8680c-995d-4a19-a213-37e20d6ae3e4", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T20:35:26Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "9bcb2df4-02f3-42d3-8bf9-10da0dd5e483", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress"} 2026-07-08T20:35:26Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "9bcb2df4-02f3-42d3-8bf9-10da0dd5e483", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c-authn-ssl"} 2026-07-08T20:35:26Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "9bcb2df4-02f3-42d3-8bf9-10da0dd5e483", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress"} 2026-07-08T20:35:26Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "e1c524ef-c06c-4574-85e1-bbb3d37f3c8e", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress"} 2026-07-08T20:35:26Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "e1c524ef-c06c-4574-85e1-bbb3d37f3c8e", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c-authn-ssl"} 2026-07-08T20:35:26Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "e1c524ef-c06c-4574-85e1-bbb3d37f3c8e", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress"} 2026-07-08T20:35:26Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "c8ab9609-8cf3-48cc-a364-2d7d6df84687", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress"} 2026-07-08T20:35:26Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "c8ab9609-8cf3-48cc-a364-2d7d6df84687", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c-authn-ssl"} 2026-07-08T20:35:26Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "c8ab9609-8cf3-48cc-a364-2d7d6df84687", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress"} 2026-07-08T20:35:26Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "7f067def-10a9-452b-855e-2fa682562e8e", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress"} 2026-07-08T20:35:26Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "7f067def-10a9-452b-855e-2fa682562e8e", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c-authn-ssl"} 2026-07-08T20:35:26Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "7f067def-10a9-452b-855e-2fa682562e8e", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress"} 2026-07-08T20:35:26Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "184d7537-5f27-4e39-aeed-673851077a87", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress"} 2026-07-08T20:35:26Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "184d7537-5f27-4e39-aeed-673851077a87", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c-authn-ssl"} 2026-07-08T20:35:26Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "184d7537-5f27-4e39-aeed-673851077a87", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress"} 2026-07-08T20:35:27Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "ee55fa85-25f3-41b3-a438-946fe0cd93a6", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress"} 2026-07-08T20:35:27Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "ee55fa85-25f3-41b3-a438-946fe0cd93a6", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c-authn-ssl"} 2026-07-08T20:35:27Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "ee55fa85-25f3-41b3-a438-946fe0cd93a6", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress"} 2026-07-08T20:35:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-b-2f232c"}, "namespace": "ai-tenant-e2e-api-b-2f232c", "name": "openshift-service-ca.crt", "reconcileID": "4e736e07-97f5-437d-acac-a415ba327a37", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-b-2f232c"} 2026-07-08T20:35:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-b-2f232c"}, "namespace": "ai-tenant-e2e-api-b-2f232c", "name": "odh-trusted-ca-bundle", "reconcileID": "770fac82-9313-4a12-9f89-376d1f68eae1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-b-2f232c"} 2026-07-08T20:35:29Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-b-2f232c"}, "namespace": "ai-tenant-e2e-api-b-2f232c", "name": "openshift-service-ca.crt", "reconcileID": "252a98e7-1116-4c62-9dfb-6edfeb7ff054", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-b-2f232c", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T20:35:29Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-b-2f232c"}, "namespace": "ai-tenant-e2e-api-b-2f232c", "name": "openshift-service-ca.crt", "reconcileID": "252a98e7-1116-4c62-9dfb-6edfeb7ff054", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-b-2f232c"} 2026-07-08T20:35:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-b-2f232c"}, "namespace": "ai-tenant-e2e-api-b-2f232c", "name": "openshift-service-ca.crt", "reconcileID": "8608684b-afbc-4cd4-bb3c-6d4e60da8d98", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-b-2f232c"} 2026-07-08T20:35:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-2f232c"}, "namespace": "ai-tenant-e2e-api-b-2f232c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d3a94cb6-6d86-4dae-a885-044e43ff3601", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-b-2f232c"} 2026-07-08T20:35:29Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-b-2f232c"}, "namespace": "ai-tenant-e2e-api-b-2f232c", "name": "odh-trusted-ca-bundle", "reconcileID": "7a770b6a-bbef-42c9-85cf-df96d2d8242c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-b-2f232c", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T20:35:29Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-b-2f232c"}, "namespace": "ai-tenant-e2e-api-b-2f232c", "name": "odh-trusted-ca-bundle", "reconcileID": "7a770b6a-bbef-42c9-85cf-df96d2d8242c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-b-2f232c"} 2026-07-08T20:35:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-2f232c"}, "namespace": "ai-tenant-e2e-api-b-2f232c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f6acdc70-ff69-497d-98f8-9b4878c8060a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-b-2f232c"} 2026-07-08T20:35:29Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "14fe526b-33f3-457d-b5c3-05e1e897631f", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress"} 2026-07-08T20:35:29Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "14fe526b-33f3-457d-b5c3-05e1e897631f", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c-authn-ssl"} 2026-07-08T20:35:29Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "14fe526b-33f3-457d-b5c3-05e1e897631f", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress"} 2026-07-08T20:35:29Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "03b1ac52-5077-4142-b667-7476c533f05a", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress"} 2026-07-08T20:35:29Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "03b1ac52-5077-4142-b667-7476c533f05a", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress", "name": "e2e-api-a-575204-authn-ssl"} 2026-07-08T20:35:29Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "03b1ac52-5077-4142-b667-7476c533f05a", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress"} 2026-07-08T20:35:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "3676b128-42e8-49ad-80a1-68d78af1edc5", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T20:35:35Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "b6b1994d-10a5-4fa5-9a3e-83a112aa86a8", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress"} 2026-07-08T20:35:35Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "b6b1994d-10a5-4fa5-9a3e-83a112aa86a8", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c-authn-ssl"} 2026-07-08T20:35:35Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "b6b1994d-10a5-4fa5-9a3e-83a112aa86a8", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress"} 2026-07-08T20:35:41Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "478e10f4-369f-4350-8071-3741edd9f989", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress"} 2026-07-08T20:35:41Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "478e10f4-369f-4350-8071-3741edd9f989", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c-authn-ssl"} 2026-07-08T20:35:41Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "478e10f4-369f-4350-8071-3741edd9f989", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress"} 2026-07-08T20:35:42Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "714c01b1-a450-4431-8bf9-bd186ddf134f", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T20:35:42Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "4932e6cd-a442-4965-bbaa-f8e02214e326", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress"} 2026-07-08T20:35:42Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "4932e6cd-a442-4965-bbaa-f8e02214e326", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress", "name": "e2e-api-a-575204-authn-ssl"} 2026-07-08T20:35:42Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "4932e6cd-a442-4965-bbaa-f8e02214e326", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress"} 2026-07-08T20:35:48Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-575204"}, "namespace": "ai-tenant-e2e-api-a-575204", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0778ca1b-bbd9-4294-af40-3bdc80e2fcf4", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-575204"} 2026-07-08T20:35:48Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-575204"}, "namespace": "ai-tenant-e2e-api-a-575204", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0778ca1b-bbd9-4294-af40-3bdc80e2fcf4", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-575204", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T20:35:48Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-575204"}, "namespace": "ai-tenant-e2e-api-a-575204", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0778ca1b-bbd9-4294-af40-3bdc80e2fcf4", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-api-a-575204 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T20:35:48Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-575204"}, "namespace": "ai-tenant-e2e-api-a-575204", "name": "odh-trusted-ca-bundle", "reconcileID": "115c1f87-a976-45e2-ae06-1ff5bbe19ee9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-575204"} 2026-07-08T20:35:48Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-575204"}, "namespace": "ai-tenant-e2e-api-a-575204", "name": "odh-trusted-ca-bundle", "reconcileID": "115c1f87-a976-45e2-ae06-1ff5bbe19ee9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-575204"} 2026-07-08T20:35:48Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-575204"}, "namespace": "ai-tenant-e2e-api-a-575204", "name": "openshift-service-ca.crt", "reconcileID": "f718312d-5d2b-4117-9464-89b8cfd7b521", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-575204"} 2026-07-08T20:35:48Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-575204"}, "namespace": "ai-tenant-e2e-api-a-575204", "name": "openshift-service-ca.crt", "reconcileID": "f718312d-5d2b-4117-9464-89b8cfd7b521", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-575204"} 2026-07-08T20:35:48Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-575204"}, "namespace": "ai-tenant-e2e-api-a-575204", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a36e0974-5725-4bab-986d-e4947d2c3bb8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-575204"} 2026-07-08T20:35:48Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-575204"}, "namespace": "ai-tenant-e2e-api-a-575204", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a36e0974-5725-4bab-986d-e4947d2c3bb8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-575204"} 2026-07-08T20:35:49Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-575204","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-575204", "reconcileID": "4b0c6907-5fb3-4def-a4f5-4fe848dc11ea", "Gateway": "e2e-api-a-575204", "namespace": "openshift-ingress"} 2026-07-08T20:35:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "a869ca43-ed7d-456e-81d2-a4cbaf8251e3", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T20:35:50Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "0b6e0e87-9edd-450a-9fcf-c55ef0072421", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress"} 2026-07-08T20:35:50Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "0b6e0e87-9edd-450a-9fcf-c55ef0072421", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c-authn-ssl"} 2026-07-08T20:35:50Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "0b6e0e87-9edd-450a-9fcf-c55ef0072421", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress"} 2026-07-08T20:35:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-2f232c"}, "namespace": "ai-tenant-e2e-api-b-2f232c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b0ea4a55-5c2b-4609-bcfa-6308ab60eb03", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-b-2f232c"} 2026-07-08T20:35:55Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-2f232c"}, "namespace": "ai-tenant-e2e-api-b-2f232c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b0ea4a55-5c2b-4609-bcfa-6308ab60eb03", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-b-2f232c", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T20:35:55Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-2f232c"}, "namespace": "ai-tenant-e2e-api-b-2f232c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b0ea4a55-5c2b-4609-bcfa-6308ab60eb03", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-api-b-2f232c because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T20:35:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-b-2f232c"}, "namespace": "ai-tenant-e2e-api-b-2f232c", "name": "odh-trusted-ca-bundle", "reconcileID": "591a81ff-502c-4ca5-b700-d609bb16a941", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-b-2f232c"} 2026-07-08T20:35:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-b-2f232c"}, "namespace": "ai-tenant-e2e-api-b-2f232c", "name": "odh-trusted-ca-bundle", "reconcileID": "591a81ff-502c-4ca5-b700-d609bb16a941", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-b-2f232c"} 2026-07-08T20:35:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-b-2f232c"}, "namespace": "ai-tenant-e2e-api-b-2f232c", "name": "openshift-service-ca.crt", "reconcileID": "b3052591-c884-4cfc-8e95-8ce81e797e6b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-b-2f232c"} 2026-07-08T20:35:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-b-2f232c"}, "namespace": "ai-tenant-e2e-api-b-2f232c", "name": "openshift-service-ca.crt", "reconcileID": "b3052591-c884-4cfc-8e95-8ce81e797e6b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-b-2f232c"} 2026-07-08T20:35:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-2f232c"}, "namespace": "ai-tenant-e2e-api-b-2f232c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8b55b47d-148f-43c2-9bf0-3138486d554b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-b-2f232c"} 2026-07-08T20:35:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-2f232c"}, "namespace": "ai-tenant-e2e-api-b-2f232c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8b55b47d-148f-43c2-9bf0-3138486d554b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-b-2f232c"} 2026-07-08T20:35:56Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-2f232c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-2f232c", "reconcileID": "b2cd7c8b-02df-4cb8-8044-a249aec86c51", "Gateway": "e2e-api-b-2f232c", "namespace": "openshift-ingress"} 2026-07-08T20:35:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "5c49fa3e-9c2c-4d5f-9f13-7fdf9eb683c8", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress"} 2026-07-08T20:35:57Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "5c49fa3e-9c2c-4d5f-9f13-7fdf9eb683c8", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309-authn-ssl"} 2026-07-08T20:35:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "5c49fa3e-9c2c-4d5f-9f13-7fdf9eb683c8", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress"} 2026-07-08T20:35:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "db4344e4-350c-46b5-9e9c-53fe59874edc", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress"} 2026-07-08T20:35:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "db4344e4-350c-46b5-9e9c-53fe59874edc", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309-authn-ssl"} 2026-07-08T20:35:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "db4344e4-350c-46b5-9e9c-53fe59874edc", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress"} 2026-07-08T20:35:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "80c2b3c2-90c5-4595-a10d-6e48f8099510", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress"} 2026-07-08T20:35:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "80c2b3c2-90c5-4595-a10d-6e48f8099510", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309-authn-ssl"} 2026-07-08T20:35:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "80c2b3c2-90c5-4595-a10d-6e48f8099510", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress"} 2026-07-08T20:35:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "43652ea6-206b-4225-8385-a2fd6f12c00f", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress"} 2026-07-08T20:35:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "43652ea6-206b-4225-8385-a2fd6f12c00f", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309-authn-ssl"} 2026-07-08T20:35:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "43652ea6-206b-4225-8385-a2fd6f12c00f", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress"} 2026-07-08T20:35:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "4917539d-f874-4fb1-9c1d-c8393dcb9866", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress"} 2026-07-08T20:35:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "4917539d-f874-4fb1-9c1d-c8393dcb9866", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309-authn-ssl"} 2026-07-08T20:35:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "4917539d-f874-4fb1-9c1d-c8393dcb9866", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress"} 2026-07-08T20:35:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "4bb1e1cf-6c80-4ec4-a394-9d79e12657b9", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress"} 2026-07-08T20:35:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "4bb1e1cf-6c80-4ec4-a394-9d79e12657b9", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309-authn-ssl"} 2026-07-08T20:35:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "4bb1e1cf-6c80-4ec4-a394-9d79e12657b9", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress"} 2026-07-08T20:35:58Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "7f4828f1-85f8-484e-abdc-4d870a15098e", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress"} 2026-07-08T20:35:58Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "7f4828f1-85f8-484e-abdc-4d870a15098e", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309-authn-ssl"} 2026-07-08T20:35:58Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "7f4828f1-85f8-484e-abdc-4d870a15098e", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress"} 2026-07-08T20:35:59Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-08T20:35:59Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-08T20:36:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-ec6309"}, "namespace": "ai-tenant-e2e-shared-a-ec6309", "name": "openshift-service-ca.crt", "reconcileID": "19d391ea-5470-42b8-a673-2794b4ea57c4", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-a-ec6309"} 2026-07-08T20:36:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-a-ec6309"}, "namespace": "ai-tenant-e2e-shared-a-ec6309", "name": "odh-trusted-ca-bundle", "reconcileID": "7eb4922a-5e17-428d-8051-14ceeac3246c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-ec6309"} 2026-07-08T20:36:00Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-ec6309"}, "namespace": "ai-tenant-e2e-shared-a-ec6309", "name": "openshift-service-ca.crt", "reconcileID": "e7c03792-4d85-49bc-a96f-cd77ed54a1a4", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-a-ec6309", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T20:36:00Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-ec6309"}, "namespace": "ai-tenant-e2e-shared-a-ec6309", "name": "openshift-service-ca.crt", "reconcileID": "e7c03792-4d85-49bc-a96f-cd77ed54a1a4", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-a-ec6309"} 2026-07-08T20:36:00Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-ec6309"}, "namespace": "ai-tenant-e2e-shared-a-ec6309", "name": "openshift-service-ca.crt", "reconcileID": "1e828382-81ba-4b87-9424-0cc31b3ebdef", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-a-ec6309", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T20:36:00Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-ec6309"}, "namespace": "ai-tenant-e2e-shared-a-ec6309", "name": "openshift-service-ca.crt", "reconcileID": "1e828382-81ba-4b87-9424-0cc31b3ebdef", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T20:36:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-ec6309"}, "namespace": "ai-tenant-e2e-shared-a-ec6309", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "227c9859-a307-4a2b-946f-70c06854ab70", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-ec6309"} 2026-07-08T20:36:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-ec6309"}, "namespace": "ai-tenant-e2e-shared-a-ec6309", "name": "openshift-service-ca.crt", "reconcileID": "fcb4c2ca-9072-40b2-97d5-2664b7b84127", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-a-ec6309"} 2026-07-08T20:36:00Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-a-ec6309"}, "namespace": "ai-tenant-e2e-shared-a-ec6309", "name": "odh-trusted-ca-bundle", "reconcileID": "5e92782f-d62a-4324-8659-eb57f2c09746", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-ec6309", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T20:36:00Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-a-ec6309"}, "namespace": "ai-tenant-e2e-shared-a-ec6309", "name": "odh-trusted-ca-bundle", "reconcileID": "5e92782f-d62a-4324-8659-eb57f2c09746", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-ec6309"} 2026-07-08T20:36:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-ec6309"}, "namespace": "ai-tenant-e2e-shared-a-ec6309", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "440844ee-c52b-46e1-b50c-05b3ed1ca16f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-ec6309"} 2026-07-08T20:36:00Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "723f1ff7-135d-4838-bb43-595b02169fc2", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress"} 2026-07-08T20:36:00Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "723f1ff7-135d-4838-bb43-595b02169fc2", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309-authn-ssl"} 2026-07-08T20:36:00Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "723f1ff7-135d-4838-bb43-595b02169fc2", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress"} 2026-07-08T20:36:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "e6150e1b-0cd5-42c3-815c-d0fa715bc302", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T20:36:05Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "9a87dfcc-b065-44bc-9104-439c8bc20e0c", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress"} 2026-07-08T20:36:05Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "9a87dfcc-b065-44bc-9104-439c8bc20e0c", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0-authn-ssl"} 2026-07-08T20:36:05Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "9a87dfcc-b065-44bc-9104-439c8bc20e0c", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress"} 2026-07-08T20:36:05Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "0fbaf166-0ade-4198-9857-1dc2b54f2644", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress"} 2026-07-08T20:36:05Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "0fbaf166-0ade-4198-9857-1dc2b54f2644", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0-authn-ssl"} 2026-07-08T20:36:05Z ERROR Reconciler error {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "0fbaf166-0ade-4198-9857-1dc2b54f2644", "error": "failed to create EnvoyFilter: could not CREATE EnvoyFilter openshift-ingress/e2e-shared-b-ca53e0-authn-ssl: envoyfilters.networking.istio.io \"e2e-shared-b-ca53e0-authn-ssl\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T20:36:05Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "f0f9fdcc-ff2c-4be6-812e-3892c02e06f5", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress"} 2026-07-08T20:36:05Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "f0f9fdcc-ff2c-4be6-812e-3892c02e06f5", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0-authn-ssl"} 2026-07-08T20:36:05Z DEBUG events Failed to reconcile EnvoyFilter: failed to create EnvoyFilter: could not CREATE EnvoyFilter openshift-ingress/e2e-shared-b-ca53e0-authn-ssl: envoyfilters.networking.istio.io "e2e-shared-b-ca53e0-authn-ssl" already exists {"type": "Warning", "object": {"kind":"Gateway","namespace":"openshift-ingress","name":"e2e-shared-b-ca53e0","uid":"2658d5ea-7720-4b40-bb35-920006ad1319","apiVersion":"gateway.networking.k8s.io/v1","resourceVersion":"50553"}, "reason": "ReconcileError"} 2026-07-08T20:36:05Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "f0f9fdcc-ff2c-4be6-812e-3892c02e06f5", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress"} 2026-07-08T20:36:05Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "d1fbb735-9d9e-4bac-955a-4e63ef0fea39", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress"} 2026-07-08T20:36:05Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "d1fbb735-9d9e-4bac-955a-4e63ef0fea39", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0-authn-ssl"} 2026-07-08T20:36:05Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "d1fbb735-9d9e-4bac-955a-4e63ef0fea39", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress"} 2026-07-08T20:36:05Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "02c8dcd5-df2f-482e-b0da-37932466a2d5", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress"} 2026-07-08T20:36:05Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "02c8dcd5-df2f-482e-b0da-37932466a2d5", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0-authn-ssl"} 2026-07-08T20:36:05Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "02c8dcd5-df2f-482e-b0da-37932466a2d5", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress"} 2026-07-08T20:36:05Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "b9ad731d-ba7b-4c81-b12e-7483daf7f2fc", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress"} 2026-07-08T20:36:05Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "b9ad731d-ba7b-4c81-b12e-7483daf7f2fc", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0-authn-ssl"} 2026-07-08T20:36:05Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "b9ad731d-ba7b-4c81-b12e-7483daf7f2fc", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress"} 2026-07-08T20:36:05Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "ab4e5d96-96b0-4426-a813-f75e2e3c8fed", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress"} 2026-07-08T20:36:05Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "ab4e5d96-96b0-4426-a813-f75e2e3c8fed", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0-authn-ssl"} 2026-07-08T20:36:05Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "ab4e5d96-96b0-4426-a813-f75e2e3c8fed", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress"} 2026-07-08T20:36:06Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-08T20:36:06Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-08T20:36:06Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "d5597adb-c991-48b0-8ebc-27f77fa2d985", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress"} 2026-07-08T20:36:06Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "d5597adb-c991-48b0-8ebc-27f77fa2d985", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0-authn-ssl"} 2026-07-08T20:36:06Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "d5597adb-c991-48b0-8ebc-27f77fa2d985", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress"} 2026-07-08T20:36:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-b-ca53e0"}, "namespace": "ai-tenant-e2e-shared-b-ca53e0", "name": "openshift-service-ca.crt", "reconcileID": "92d81a36-3ef2-4378-ac85-efbc36119170", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-b-ca53e0"} 2026-07-08T20:36:08Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-b-ca53e0"}, "namespace": "ai-tenant-e2e-shared-b-ca53e0", "name": "openshift-service-ca.crt", "reconcileID": "c138f47a-8c3c-48ec-8c1b-85b467fee9c7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-b-ca53e0", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T20:36:08Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-b-ca53e0"}, "namespace": "ai-tenant-e2e-shared-b-ca53e0", "name": "openshift-service-ca.crt", "reconcileID": "c138f47a-8c3c-48ec-8c1b-85b467fee9c7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-b-ca53e0"} 2026-07-08T20:36:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-b-ca53e0"}, "namespace": "ai-tenant-e2e-shared-b-ca53e0", "name": "odh-trusted-ca-bundle", "reconcileID": "10858d9f-34fd-4700-b6f9-bdaec5535715", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-ca53e0"} 2026-07-08T20:36:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-ca53e0"}, "namespace": "ai-tenant-e2e-shared-b-ca53e0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "497076e2-e6ca-4f89-99ec-0be3389b55ba", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-ca53e0"} 2026-07-08T20:36:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-b-ca53e0"}, "namespace": "ai-tenant-e2e-shared-b-ca53e0", "name": "openshift-service-ca.crt", "reconcileID": "f4b4eb31-c123-4919-bd11-79290f7e42d9", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-b-ca53e0"} 2026-07-08T20:36:08Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-b-ca53e0"}, "namespace": "ai-tenant-e2e-shared-b-ca53e0", "name": "odh-trusted-ca-bundle", "reconcileID": "ee31e873-9539-437c-83a9-b891eb782aaa", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-ca53e0", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T20:36:08Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-b-ca53e0"}, "namespace": "ai-tenant-e2e-shared-b-ca53e0", "name": "odh-trusted-ca-bundle", "reconcileID": "ee31e873-9539-437c-83a9-b891eb782aaa", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-ca53e0"} 2026-07-08T20:36:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-ca53e0"}, "namespace": "ai-tenant-e2e-shared-b-ca53e0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b5ba592b-9f32-4428-8b7c-083c4a44c778", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-ca53e0"} 2026-07-08T20:36:08Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "7ad2566c-bf5c-4686-bc26-022057e42426", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress"} 2026-07-08T20:36:08Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "7ad2566c-bf5c-4686-bc26-022057e42426", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0-authn-ssl"} 2026-07-08T20:36:08Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "7ad2566c-bf5c-4686-bc26-022057e42426", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress"} 2026-07-08T20:36:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "e2e769ef-7ba9-4e57-a878-2e658fe3b0b8", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T20:36:09Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "382a97fe-846f-4b41-8d50-873f6caa64f6", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress"} 2026-07-08T20:36:09Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "382a97fe-846f-4b41-8d50-873f6caa64f6", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309-authn-ssl"} 2026-07-08T20:36:09Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "382a97fe-846f-4b41-8d50-873f6caa64f6", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress"} 2026-07-08T20:36:16Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "88527164-5bf7-42d6-832e-ee80fea5f24c", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress"} 2026-07-08T20:36:16Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "88527164-5bf7-42d6-832e-ee80fea5f24c", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0-authn-ssl"} 2026-07-08T20:36:16Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "88527164-5bf7-42d6-832e-ee80fea5f24c", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress"} 2026-07-08T20:36:21Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "14f2b5bd-b214-41a5-b0dd-0c495eb1bc83", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T20:36:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "50a2cc95-2e30-4478-a57d-69797a999199", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T20:39:18Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "ba633de0-91fe-4630-8129-92c50740fb61", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T20:39:19Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "a7f93073-99ec-44c3-9a4a-17770c7d7028", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress"} 2026-07-08T20:39:19Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "a7f93073-99ec-44c3-9a4a-17770c7d7028", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309-authn-ssl"} 2026-07-08T20:39:19Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "a7f93073-99ec-44c3-9a4a-17770c7d7028", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress"} 2026-07-08T20:39:23Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-ec6309"}, "namespace": "ai-tenant-e2e-shared-a-ec6309", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fbaf61cd-6746-4910-9e7e-af615094a38b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-ec6309"} 2026-07-08T20:39:23Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-ec6309"}, "namespace": "ai-tenant-e2e-shared-a-ec6309", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fbaf61cd-6746-4910-9e7e-af615094a38b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-ec6309", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T20:39:23Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-ec6309"}, "namespace": "ai-tenant-e2e-shared-a-ec6309", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fbaf61cd-6746-4910-9e7e-af615094a38b", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-ec6309 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T20:39:23Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-a-ec6309"}, "namespace": "ai-tenant-e2e-shared-a-ec6309", "name": "odh-trusted-ca-bundle", "reconcileID": "e4c5a773-1821-44be-ae5e-a1bd519ad0e4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-ec6309"} 2026-07-08T20:39:23Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-a-ec6309"}, "namespace": "ai-tenant-e2e-shared-a-ec6309", "name": "odh-trusted-ca-bundle", "reconcileID": "e4c5a773-1821-44be-ae5e-a1bd519ad0e4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-ec6309"} 2026-07-08T20:39:23Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-ec6309"}, "namespace": "ai-tenant-e2e-shared-a-ec6309", "name": "openshift-service-ca.crt", "reconcileID": "a85d5502-1f01-4cf1-b961-920e1ecf4b0d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-a-ec6309"} 2026-07-08T20:39:23Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-ec6309"}, "namespace": "ai-tenant-e2e-shared-a-ec6309", "name": "openshift-service-ca.crt", "reconcileID": "a85d5502-1f01-4cf1-b961-920e1ecf4b0d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-a-ec6309"} 2026-07-08T20:39:23Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-ec6309"}, "namespace": "ai-tenant-e2e-shared-a-ec6309", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f410539d-b55b-457b-895b-a14972babb71", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-ec6309"} 2026-07-08T20:39:23Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-ec6309"}, "namespace": "ai-tenant-e2e-shared-a-ec6309", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f410539d-b55b-457b-895b-a14972babb71", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-ec6309"} 2026-07-08T20:39:24Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-ec6309","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-ec6309", "reconcileID": "6287819a-9230-4336-851c-8f9b9f7fc86e", "Gateway": "e2e-shared-a-ec6309", "namespace": "openshift-ingress"} 2026-07-08T20:39:25Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "4f5a5073-eae2-444a-a70e-a9c7bd2c6a6b", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T20:39:26Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "e9995e60-0807-4c58-b5dd-1dfea8758996", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress"} 2026-07-08T20:39:26Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "e9995e60-0807-4c58-b5dd-1dfea8758996", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0-authn-ssl"} 2026-07-08T20:39:26Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "e9995e60-0807-4c58-b5dd-1dfea8758996", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress"} 2026-07-08T20:39:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-ca53e0"}, "namespace": "ai-tenant-e2e-shared-b-ca53e0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "051054e3-9dd6-4525-9de6-6459cce51f33", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-ca53e0"} 2026-07-08T20:39:30Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-ca53e0"}, "namespace": "ai-tenant-e2e-shared-b-ca53e0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "051054e3-9dd6-4525-9de6-6459cce51f33", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-ca53e0", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T20:39:30Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-ca53e0"}, "namespace": "ai-tenant-e2e-shared-b-ca53e0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "051054e3-9dd6-4525-9de6-6459cce51f33", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-ca53e0 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T20:39:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-b-ca53e0"}, "namespace": "ai-tenant-e2e-shared-b-ca53e0", "name": "odh-trusted-ca-bundle", "reconcileID": "8c1a0a20-bb23-4617-b8d8-75807d20c914", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-ca53e0"} 2026-07-08T20:39:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-b-ca53e0"}, "namespace": "ai-tenant-e2e-shared-b-ca53e0", "name": "odh-trusted-ca-bundle", "reconcileID": "8c1a0a20-bb23-4617-b8d8-75807d20c914", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-ca53e0"} 2026-07-08T20:39:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-b-ca53e0"}, "namespace": "ai-tenant-e2e-shared-b-ca53e0", "name": "openshift-service-ca.crt", "reconcileID": "2cbd8cac-9b82-46ea-a7db-8041f154a51a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-b-ca53e0"} 2026-07-08T20:39:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-b-ca53e0"}, "namespace": "ai-tenant-e2e-shared-b-ca53e0", "name": "openshift-service-ca.crt", "reconcileID": "2cbd8cac-9b82-46ea-a7db-8041f154a51a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-b-ca53e0"} 2026-07-08T20:39:30Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-ca53e0"}, "namespace": "ai-tenant-e2e-shared-b-ca53e0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4ff7b3be-4555-457c-93a4-90402d20a552", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-ca53e0"} 2026-07-08T20:39:30Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-ca53e0"}, "namespace": "ai-tenant-e2e-shared-b-ca53e0", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4ff7b3be-4555-457c-93a4-90402d20a552", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-ca53e0"} 2026-07-08T20:39:31Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-ca53e0","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-ca53e0", "reconcileID": "31e0269f-3949-40a7-b9c7-d233ba73867e", "Gateway": "e2e-shared-b-ca53e0", "namespace": "openshift-ingress"}