2026-06-03T20:52:25Z INFO Starting workers {"controller": "inferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "InferenceService", "worker count": 1} 2026-06-03T20:52:25Z INFO Starting workers {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "worker count": 1} 2026-06-03T20:52:25Z INFO Starting workers {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "worker count": 1} 2026-06-03T20:52:26Z ERROR fail to get ray ca cert secret {"ResourceName": "openshift-monitoring", "Namespace": "openshift-monitoring", "secret": "ray-ca-tls", "namespace": "opendatahub", "error": "Secret \"ray-ca-tls\" not found"} github.com/opendatahub-io/odh-model-controller/internal/controller/serving.(*ServingRuntimeReconciler).reconcileMultiNodeSR /workspace/internal/controller/serving/servingruntime_controller.go:289 github.com/opendatahub-io/odh-model-controller/internal/controller/serving.(*ServingRuntimeReconciler).Reconcile /workspace/internal/controller/serving/servingruntime_controller.go:270 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-03T20:52:26Z ERROR Reconciler error {"controller": "servingruntime", "controllerGroup": "serving.kserve.io", "controllerKind": "ServingRuntime", "ServingRuntime": {"name":"openshift-monitoring","namespace":"openshift-monitoring"}, "namespace": "openshift-monitoring", "name": "openshift-monitoring", "reconcileID": "24305016-0d33-4ed8-8c11-e025f91bcc33", "error": "Secret \"ray-ca-tls\" not found"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-03T20:52:26Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-03T20:52:26Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-03T20:52:27Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-03T20:52:27Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-03T20:52:27Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "openshift-monitoring", "Namespace": "openshift-monitoring"} 2026-06-03T20:52:27Z INFO Multi Node reconciling. {"ResourceName": "openshift-monitoring", "Namespace": "openshift-monitoring"} 2026-06-03T20:52:29Z INFO Multi Node reconciled successfully. {"ResourceName": "openshift-monitoring", "Namespace": "openshift-monitoring"} 2026-06-03T20:53:10Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"kuadrant-system"}, "namespace": "kuadrant-system", "name": "odh-trusted-ca-bundle", "reconcileID": "e49dc435-b52b-4451-8e2a-45002c27daf4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "kuadrant-system"} 2026-06-03T20:53:10Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"kuadrant-system"}, "namespace": "kuadrant-system", "name": "openshift-service-ca.crt", "reconcileID": "466b0560-3bd1-4a85-96c4-fadd1b0bf65a", "ConfigMap": "openshift-service-ca.crt", "namespace": "kuadrant-system"} 2026-06-03T20:53:10Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"kuadrant-system"}, "namespace": "kuadrant-system", "name": "openshift-service-ca.crt", "reconcileID": "64a60315-ab27-45b0-8a45-872b8e227c6a", "ConfigMap": "openshift-service-ca.crt", "namespace": "kuadrant-system", "create": "odh-kserve-custom-ca-bundle"} 2026-06-03T20:53:10Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"kuadrant-system"}, "namespace": "kuadrant-system", "name": "openshift-service-ca.crt", "reconcileID": "64a60315-ab27-45b0-8a45-872b8e227c6a", "ConfigMap": "openshift-service-ca.crt", "namespace": "kuadrant-system"} 2026-06-03T20:53:10Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"kuadrant-system"}, "namespace": "kuadrant-system", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d8ca10c1-839e-4099-a7cb-b97f91ace40d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "kuadrant-system"} 2026-06-03T20:53:10Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"kuadrant-system"}, "namespace": "kuadrant-system", "name": "openshift-service-ca.crt", "reconcileID": "bd7b957a-1fa8-4eca-937a-b51f6f543b85", "ConfigMap": "openshift-service-ca.crt", "namespace": "kuadrant-system"} 2026-06-03T20:53:10Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"kuadrant-system"}, "namespace": "kuadrant-system", "name": "odh-trusted-ca-bundle", "reconcileID": "9a0b7e97-8dd8-44d7-8d38-9e6975fd3ace", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "kuadrant-system", "update": "odh-kserve-custom-ca-bundle"} 2026-06-03T20:53:10Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"kuadrant-system"}, "namespace": "kuadrant-system", "name": "odh-trusted-ca-bundle", "reconcileID": "9a0b7e97-8dd8-44d7-8d38-9e6975fd3ace", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "kuadrant-system"} 2026-06-03T20:53:10Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"kuadrant-system"}, "namespace": "kuadrant-system", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "09512f4f-61aa-45be-b4fa-83da3e34de3f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "kuadrant-system"} 2026-06-03T20:54:35Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "bd8b6c15-70a3-4c77-8c84-9e923b337800", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:54:36Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "bd8b6c15-70a3-4c77-8c84-9e923b337800", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T20:54:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "bd8b6c15-70a3-4c77-8c84-9e923b337800", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:54:36Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "e103b85f-f249-4664-8fbd-195a117503cc", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:54:36Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "e103b85f-f249-4664-8fbd-195a117503cc", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T20:54:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "e103b85f-f249-4664-8fbd-195a117503cc", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:54:37Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "5cc03036-177c-4599-8c02-6de59cdbbe1c", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:54:37Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "5cc03036-177c-4599-8c02-6de59cdbbe1c", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T20:54:37Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "5cc03036-177c-4599-8c02-6de59cdbbe1c", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:54:40Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "f07195c1-ec51-4f3e-a339-f5a3f373da53", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:54:40Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "f07195c1-ec51-4f3e-a339-f5a3f373da53", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T20:54:40Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "f07195c1-ec51-4f3e-a339-f5a3f373da53", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:55:14Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"keycloak-system"}, "namespace": "keycloak-system", "name": "openshift-service-ca.crt", "reconcileID": "e220dd7b-080e-473f-8c67-3e7b4fae5240", "ConfigMap": "openshift-service-ca.crt", "namespace": "keycloak-system"} 2026-06-03T20:55:14Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"keycloak-system"}, "namespace": "keycloak-system", "name": "openshift-service-ca.crt", "reconcileID": "070f749b-3535-441c-b139-7b8183aa3864", "ConfigMap": "openshift-service-ca.crt", "namespace": "keycloak-system", "create": "odh-kserve-custom-ca-bundle"} 2026-06-03T20:55:14Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"keycloak-system"}, "namespace": "keycloak-system", "name": "openshift-service-ca.crt", "reconcileID": "070f749b-3535-441c-b139-7b8183aa3864", "ConfigMap": "openshift-service-ca.crt", "namespace": "keycloak-system"} 2026-06-03T20:55:14Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"keycloak-system"}, "namespace": "keycloak-system", "name": "odh-trusted-ca-bundle", "reconcileID": "33d94e25-f286-4c43-bcea-420a2d9b39f1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "keycloak-system"} 2026-06-03T20:55:14Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"keycloak-system"}, "namespace": "keycloak-system", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "31613627-eb9e-4921-b800-ba97df19cf9a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "keycloak-system"} 2026-06-03T20:55:14Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"keycloak-system"}, "namespace": "keycloak-system", "name": "openshift-service-ca.crt", "reconcileID": "9bad24fc-be2d-47b1-99f4-9175820cbd77", "ConfigMap": "openshift-service-ca.crt", "namespace": "keycloak-system"} 2026-06-03T20:55:14Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"keycloak-system"}, "namespace": "keycloak-system", "name": "odh-trusted-ca-bundle", "reconcileID": "d62f73b4-e458-4967-b766-7bf6041960ad", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "keycloak-system", "update": "odh-kserve-custom-ca-bundle"} 2026-06-03T20:55:14Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"keycloak-system"}, "namespace": "keycloak-system", "name": "odh-trusted-ca-bundle", "reconcileID": "d62f73b4-e458-4967-b766-7bf6041960ad", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "keycloak-system"} 2026-06-03T20:55:14Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"keycloak-system"}, "namespace": "keycloak-system", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "db7b8e2f-d3e3-4417-b1d3-5411c9bb94dd", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "keycloak-system"} 2026-06-03T20:56:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"models-as-a-service"}, "namespace": "models-as-a-service", "name": "openshift-service-ca.crt", "reconcileID": "c629803f-b52b-441a-bdb3-f52993ca02c7", "ConfigMap": "openshift-service-ca.crt", "namespace": "models-as-a-service"} 2026-06-03T20:56:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"models-as-a-service"}, "namespace": "models-as-a-service", "name": "odh-trusted-ca-bundle", "reconcileID": "528917a3-fac9-400d-a46f-f3137ed1e115", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "models-as-a-service"} 2026-06-03T20:56:17Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"models-as-a-service"}, "namespace": "models-as-a-service", "name": "openshift-service-ca.crt", "reconcileID": "700f1979-d612-435c-ab5c-d585f0bbc093", "ConfigMap": "openshift-service-ca.crt", "namespace": "models-as-a-service", "create": "odh-kserve-custom-ca-bundle"} 2026-06-03T20:56:17Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"models-as-a-service"}, "namespace": "models-as-a-service", "name": "openshift-service-ca.crt", "reconcileID": "700f1979-d612-435c-ab5c-d585f0bbc093", "ConfigMap": "openshift-service-ca.crt", "namespace": "models-as-a-service"} 2026-06-03T20:56:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenants"}, "namespace": "ai-tenants", "name": "openshift-service-ca.crt", "reconcileID": "68254b91-85b6-4bdd-9ff8-fb0cf5ca6673", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenants"} 2026-06-03T20:56:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"models-as-a-service"}, "namespace": "models-as-a-service", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4b11b5af-c3bc-40cf-967c-4ca1c1fb6cca", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "models-as-a-service"} 2026-06-03T20:56:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"models-as-a-service"}, "namespace": "models-as-a-service", "name": "openshift-service-ca.crt", "reconcileID": "f3185508-87ef-40eb-9043-9d9ab6c03851", "ConfigMap": "openshift-service-ca.crt", "namespace": "models-as-a-service"} 2026-06-03T20:56:17Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenants"}, "namespace": "ai-tenants", "name": "openshift-service-ca.crt", "reconcileID": "6baf9896-712a-4609-893c-64736e974d7d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenants", "create": "odh-kserve-custom-ca-bundle"} 2026-06-03T20:56:17Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenants"}, "namespace": "ai-tenants", "name": "openshift-service-ca.crt", "reconcileID": "6baf9896-712a-4609-893c-64736e974d7d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenants"} 2026-06-03T20:56:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenants"}, "namespace": "ai-tenants", "name": "odh-trusted-ca-bundle", "reconcileID": "4b3557cf-5a07-479f-90be-8cccd433720f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenants"} 2026-06-03T20:56:17Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"models-as-a-service"}, "namespace": "models-as-a-service", "name": "odh-trusted-ca-bundle", "reconcileID": "bf1d7809-f693-4641-9646-b58c0b3ea66b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "models-as-a-service", "update": "odh-kserve-custom-ca-bundle"} 2026-06-03T20:56:17Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"models-as-a-service"}, "namespace": "models-as-a-service", "name": "odh-trusted-ca-bundle", "reconcileID": "bf1d7809-f693-4641-9646-b58c0b3ea66b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "models-as-a-service"} 2026-06-03T20:56:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenants"}, "namespace": "ai-tenants", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "000e84bf-23c2-4014-8e5d-3dab8b620f2f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenants"} 2026-06-03T20:56:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenants"}, "namespace": "ai-tenants", "name": "openshift-service-ca.crt", "reconcileID": "9dbc1107-aa80-470d-a626-c50781e879e5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenants"} 2026-06-03T20:56:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"models-as-a-service"}, "namespace": "models-as-a-service", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a3573384-3968-429d-a122-24de8b01266b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "models-as-a-service"} 2026-06-03T20:56:17Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenants"}, "namespace": "ai-tenants", "name": "odh-trusted-ca-bundle", "reconcileID": "f72304fc-0477-4c68-9e71-0693e6b69a51", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenants", "update": "odh-kserve-custom-ca-bundle"} 2026-06-03T20:56:17Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenants"}, "namespace": "ai-tenants", "name": "odh-trusted-ca-bundle", "reconcileID": "f72304fc-0477-4c68-9e71-0693e6b69a51", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenants"} 2026-06-03T20:56:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenants"}, "namespace": "ai-tenants", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "aefc5550-0780-4978-bc14-bc296c82a338", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenants"} 2026-06-03T20:56:48Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "580b98a5-4758-4ef5-be21-57e3ec050b65", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:56:48Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "580b98a5-4758-4ef5-be21-57e3ec050b65", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T20:56:48Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "580b98a5-4758-4ef5-be21-57e3ec050b65", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:56:49Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "a8ffa7f8-6922-447d-b432-8f4ecf9be8cd", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:56:49Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "a8ffa7f8-6922-447d-b432-8f4ecf9be8cd", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T20:56:49Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "a8ffa7f8-6922-447d-b432-8f4ecf9be8cd", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:57:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"llm"}, "namespace": "llm", "name": "openshift-service-ca.crt", "reconcileID": "798333d6-ec54-4cf1-b30b-9367640d373c", "ConfigMap": "openshift-service-ca.crt", "namespace": "llm"} 2026-06-03T20:57:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"llm"}, "namespace": "llm", "name": "odh-trusted-ca-bundle", "reconcileID": "df1cc919-af9b-46c2-b673-4984ec591bf0", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "llm"} 2026-06-03T20:57:40Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"llm"}, "namespace": "llm", "name": "openshift-service-ca.crt", "reconcileID": "fc25d563-34de-4bdb-bc63-0de79df0f7a2", "ConfigMap": "openshift-service-ca.crt", "namespace": "llm", "create": "odh-kserve-custom-ca-bundle"} 2026-06-03T20:57:40Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"llm"}, "namespace": "llm", "name": "openshift-service-ca.crt", "reconcileID": "fc25d563-34de-4bdb-bc63-0de79df0f7a2", "ConfigMap": "openshift-service-ca.crt", "namespace": "llm"} 2026-06-03T20:57:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"llm"}, "namespace": "llm", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2fec40f6-7b56-45df-b188-2cce0d59d803", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "llm"} 2026-06-03T20:57:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"llm"}, "namespace": "llm", "name": "openshift-service-ca.crt", "reconcileID": "2352361f-122c-4fbe-bfbf-61553ae5fb93", "ConfigMap": "openshift-service-ca.crt", "namespace": "llm"} 2026-06-03T20:57:40Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"llm"}, "namespace": "llm", "name": "odh-trusted-ca-bundle", "reconcileID": "2e181755-7da5-49e9-b1b6-e83f400dd0bc", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "llm", "update": "odh-kserve-custom-ca-bundle"} 2026-06-03T20:57:40Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"llm"}, "namespace": "llm", "name": "odh-trusted-ca-bundle", "reconcileID": "2e181755-7da5-49e9-b1b6-e83f400dd0bc", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "llm"} 2026-06-03T20:57:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"llm"}, "namespace": "llm", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "38ae3927-f677-47c7-8e5a-853383ae0b7d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "f8ed53bc-67a6-4a78-8839-080646198fc0", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "f8ed53bc-67a6-4a78-8839-080646198fc0", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "f8ed53bc-67a6-4a78-8839-080646198fc0", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "c5be74bc-044f-418c-9c7b-13f77d16eb06", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:57:41Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "c5be74bc-044f-418c-9c7b-13f77d16eb06", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T20:57:41Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "f8ed53bc-67a6-4a78-8839-080646198fc0", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm", "name": "e2e-distinct-2-simulated-kserve-route-authn"} 2026-06-03T20:57:41Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "f8ed53bc-67a6-4a78-8839-080646198fc0", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-03T20:57:41Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "f8ed53bc-67a6-4a78-8839-080646198fc0", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "c5be74bc-044f-418c-9c7b-13f77d16eb06", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:57:41Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "5dc1f959-3a02-464b-8be5-00e1da04705d", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "5dc1f959-3a02-464b-8be5-00e1da04705d", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "5dc1f959-3a02-464b-8be5-00e1da04705d", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0228cae2-911f-4dd7-b615-a2ca518359dd", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:57:41Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "5dc1f959-3a02-464b-8be5-00e1da04705d", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm", "name": "e2e-distinct-simulated-kserve-route-authn"} 2026-06-03T20:57:41Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "5dc1f959-3a02-464b-8be5-00e1da04705d", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-03T20:57:41Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "5dc1f959-3a02-464b-8be5-00e1da04705d", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0228cae2-911f-4dd7-b615-a2ca518359dd", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T20:57:41Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0228cae2-911f-4dd7-b615-a2ca518359dd", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:57:41Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "2216f75f-0e4a-41d8-9814-72c72b350195", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "2216f75f-0e4a-41d8-9814-72c72b350195", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "2216f75f-0e4a-41d8-9814-72c72b350195", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "2216f75f-0e4a-41d8-9814-72c72b350195", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm", "name": "e2e-distinct-2-simulated-kserve-route-authn"} 2026-06-03T20:57:41Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "2216f75f-0e4a-41d8-9814-72c72b350195", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-03T20:57:41Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "2216f75f-0e4a-41d8-9814-72c72b350195", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "bcf10eac-d7b8-4666-9d38-8d87aeeb76b4", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "bcf10eac-d7b8-4666-9d38-8d87aeeb76b4", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "bcf10eac-d7b8-4666-9d38-8d87aeeb76b4", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "2b2c46ea-898d-43c8-9a0f-5be4c318544a", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:57:41Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "bcf10eac-d7b8-4666-9d38-8d87aeeb76b4", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm", "name": "e2e-trlp-test-simulated-kserve-route-authn"} 2026-06-03T20:57:41Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "bcf10eac-d7b8-4666-9d38-8d87aeeb76b4", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-03T20:57:41Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "bcf10eac-d7b8-4666-9d38-8d87aeeb76b4", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "2b2c46ea-898d-43c8-9a0f-5be4c318544a", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T20:57:41Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "2b2c46ea-898d-43c8-9a0f-5be4c318544a", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:57:41Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "a8a08903-b08d-40d7-9cb9-6acf90b343b0", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "a8a08903-b08d-40d7-9cb9-6acf90b343b0", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "a8a08903-b08d-40d7-9cb9-6acf90b343b0", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "d08aba5b-4b06-4854-9d66-87d6227b81e3", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:57:41Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "a8a08903-b08d-40d7-9cb9-6acf90b343b0", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated-kserve-route-authn"} 2026-06-03T20:57:41Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "a8a08903-b08d-40d7-9cb9-6acf90b343b0", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-03T20:57:41Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "a8a08903-b08d-40d7-9cb9-6acf90b343b0", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "d08aba5b-4b06-4854-9d66-87d6227b81e3", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T20:57:41Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "d08aba5b-4b06-4854-9d66-87d6227b81e3", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:57:41Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "30e99b5d-f793-4454-a3ec-f7927c269df7", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "30e99b5d-f793-4454-a3ec-f7927c269df7", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "30e99b5d-f793-4454-a3ec-f7927c269df7", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "c43c7726-b5be-48d0-beef-bbb75d52069d", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:57:41Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "30e99b5d-f793-4454-a3ec-f7927c269df7", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm", "name": "facebook-opt-125m-simulated-kserve-route-authn"} 2026-06-03T20:57:41Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "30e99b5d-f793-4454-a3ec-f7927c269df7", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-03T20:57:41Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "30e99b5d-f793-4454-a3ec-f7927c269df7", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:57:41Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "c43c7726-b5be-48d0-beef-bbb75d52069d", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T20:57:41Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "c43c7726-b5be-48d0-beef-bbb75d52069d", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:57:41Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "046559f2-b195-4bfa-ac0f-c506652b5f52", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "046559f2-b195-4bfa-ac0f-c506652b5f52", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "046559f2-b195-4bfa-ac0f-c506652b5f52", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-03T20:57:41Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "6c617b6a-d786-4f82-b58e-31871efa488f", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:57:41Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "046559f2-b195-4bfa-ac0f-c506652b5f52", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm", "name": "premium-simulated-simulated-premium-kserve-route-authn"} 2026-06-03T20:57:41Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "046559f2-b195-4bfa-ac0f-c506652b5f52", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm", "authType": "userdefined"} 2026-06-03T20:57:41Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "046559f2-b195-4bfa-ac0f-c506652b5f52", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-03T20:57:41Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "6c617b6a-d786-4f82-b58e-31871efa488f", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T20:57:41Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "6c617b6a-d786-4f82-b58e-31871efa488f", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:57:46Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0f6cfe4a-4995-405d-af95-535177e3090a", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:57:46Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0f6cfe4a-4995-405d-af95-535177e3090a", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T20:57:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0f6cfe4a-4995-405d-af95-535177e3090a", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:57:47Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "691c6f08-ace1-44d1-a943-2d95216abbe4", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-03T20:57:47Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "691c6f08-ace1-44d1-a943-2d95216abbe4", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-03T20:57:47Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "691c6f08-ace1-44d1-a943-2d95216abbe4", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-03T20:57:47Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "691c6f08-ace1-44d1-a943-2d95216abbe4", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm", "name": "e2e-distinct-2-simulated-kserve-route-authn"} 2026-06-03T20:57:47Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "691c6f08-ace1-44d1-a943-2d95216abbe4", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-03T20:57:47Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "691c6f08-ace1-44d1-a943-2d95216abbe4", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-03T20:57:47Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "015b6601-05b3-43ff-a522-995903a07362", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-03T20:57:47Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "015b6601-05b3-43ff-a522-995903a07362", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-03T20:57:47Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "015b6601-05b3-43ff-a522-995903a07362", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-03T20:57:47Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "015b6601-05b3-43ff-a522-995903a07362", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm", "name": "e2e-distinct-simulated-kserve-route-authn"} 2026-06-03T20:57:47Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "015b6601-05b3-43ff-a522-995903a07362", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-03T20:57:47Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "015b6601-05b3-43ff-a522-995903a07362", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-03T20:57:54Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "d30b65d4-ded4-4165-bd94-540df7d1bb2e", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:57:54Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "d30b65d4-ded4-4165-bd94-540df7d1bb2e", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T20:57:54Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "d30b65d4-ded4-4165-bd94-540df7d1bb2e", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:57:55Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "06fd076b-f1d3-4bff-bc69-6c6003dc59a8", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-03T20:57:55Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "06fd076b-f1d3-4bff-bc69-6c6003dc59a8", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-03T20:57:55Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "06fd076b-f1d3-4bff-bc69-6c6003dc59a8", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-03T20:57:55Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "06fd076b-f1d3-4bff-bc69-6c6003dc59a8", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm", "name": "e2e-distinct-simulated-kserve-route-authn"} 2026-06-03T20:57:55Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "06fd076b-f1d3-4bff-bc69-6c6003dc59a8", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-03T20:57:55Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "06fd076b-f1d3-4bff-bc69-6c6003dc59a8", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-03T20:57:55Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "db96fdaf-496f-4db7-9c12-518a1ca6664b", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-03T20:57:55Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "db96fdaf-496f-4db7-9c12-518a1ca6664b", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-03T20:57:55Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "db96fdaf-496f-4db7-9c12-518a1ca6664b", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-03T20:57:55Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "db96fdaf-496f-4db7-9c12-518a1ca6664b", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm", "name": "e2e-trlp-test-simulated-kserve-route-authn"} 2026-06-03T20:57:55Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "db96fdaf-496f-4db7-9c12-518a1ca6664b", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-03T20:57:55Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "db96fdaf-496f-4db7-9c12-518a1ca6664b", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-03T20:58:07Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "937941f8-d8be-41a3-a422-915d14f0dd41", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:58:07Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "937941f8-d8be-41a3-a422-915d14f0dd41", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T20:58:07Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "937941f8-d8be-41a3-a422-915d14f0dd41", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:58:07Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "72afaa7a-721b-49e2-bf15-6983e2d0e29d", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-03T20:58:07Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "72afaa7a-721b-49e2-bf15-6983e2d0e29d", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-03T20:58:07Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "72afaa7a-721b-49e2-bf15-6983e2d0e29d", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-03T20:58:07Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "72afaa7a-721b-49e2-bf15-6983e2d0e29d", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm", "name": "e2e-trlp-test-simulated-kserve-route-authn"} 2026-06-03T20:58:07Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "72afaa7a-721b-49e2-bf15-6983e2d0e29d", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-03T20:58:07Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "72afaa7a-721b-49e2-bf15-6983e2d0e29d", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-03T20:58:07Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "e0af16a5-3de2-481d-8f58-e3f6a2e64dea", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:58:07Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "e0af16a5-3de2-481d-8f58-e3f6a2e64dea", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:58:07Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "e0af16a5-3de2-481d-8f58-e3f6a2e64dea", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:58:07Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "e0af16a5-3de2-481d-8f58-e3f6a2e64dea", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated-kserve-route-authn"} 2026-06-03T20:58:07Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "e0af16a5-3de2-481d-8f58-e3f6a2e64dea", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-03T20:58:07Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "e0af16a5-3de2-481d-8f58-e3f6a2e64dea", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:58:31Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "4e05aa19-6bfe-4492-8799-f7264489dfa7", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:58:31Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "4e05aa19-6bfe-4492-8799-f7264489dfa7", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T20:58:31Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "4e05aa19-6bfe-4492-8799-f7264489dfa7", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:58:32Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "1693bb17-b0eb-4699-a643-d4209b788084", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:58:32Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "1693bb17-b0eb-4699-a643-d4209b788084", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:58:32Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "1693bb17-b0eb-4699-a643-d4209b788084", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:58:32Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "1693bb17-b0eb-4699-a643-d4209b788084", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated-kserve-route-authn"} 2026-06-03T20:58:32Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "1693bb17-b0eb-4699-a643-d4209b788084", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-03T20:58:32Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "1693bb17-b0eb-4699-a643-d4209b788084", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:58:32Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "f3a8b82f-b2a8-4921-9eaa-0f7a5f22bb1a", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:58:32Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "f3a8b82f-b2a8-4921-9eaa-0f7a5f22bb1a", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:58:32Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "f3a8b82f-b2a8-4921-9eaa-0f7a5f22bb1a", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:58:32Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "f3a8b82f-b2a8-4921-9eaa-0f7a5f22bb1a", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm", "name": "facebook-opt-125m-simulated-kserve-route-authn"} 2026-06-03T20:58:32Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "f3a8b82f-b2a8-4921-9eaa-0f7a5f22bb1a", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-03T20:58:32Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "f3a8b82f-b2a8-4921-9eaa-0f7a5f22bb1a", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:58:38Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "e302f905-e24c-4f2a-a818-f9b333de28de", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:58:38Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "e302f905-e24c-4f2a-a818-f9b333de28de", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T20:58:38Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "e302f905-e24c-4f2a-a818-f9b333de28de", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:58:39Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "69e394a6-7340-432c-b52c-a165d2b2b0ba", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:58:39Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "69e394a6-7340-432c-b52c-a165d2b2b0ba", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:58:39Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "69e394a6-7340-432c-b52c-a165d2b2b0ba", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:58:39Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "69e394a6-7340-432c-b52c-a165d2b2b0ba", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm", "name": "facebook-opt-125m-simulated-kserve-route-authn"} 2026-06-03T20:58:39Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "69e394a6-7340-432c-b52c-a165d2b2b0ba", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-03T20:58:39Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "69e394a6-7340-432c-b52c-a165d2b2b0ba", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:58:39Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "b1aef88f-80b0-4f3f-a622-6fcde67b28c1", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-03T20:58:39Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "b1aef88f-80b0-4f3f-a622-6fcde67b28c1", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-03T20:58:39Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "b1aef88f-80b0-4f3f-a622-6fcde67b28c1", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-03T20:58:39Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "b1aef88f-80b0-4f3f-a622-6fcde67b28c1", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm", "name": "premium-simulated-simulated-premium-kserve-route-authn"} 2026-06-03T20:58:39Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "b1aef88f-80b0-4f3f-a622-6fcde67b28c1", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm", "authType": "userdefined"} 2026-06-03T20:58:39Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "b1aef88f-80b0-4f3f-a622-6fcde67b28c1", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-03T20:58:54Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "2388f1cd-6f22-4d9b-bfc5-e63f215c768a", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:58:54Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "2388f1cd-6f22-4d9b-bfc5-e63f215c768a", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T20:58:54Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "2388f1cd-6f22-4d9b-bfc5-e63f215c768a", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T20:58:55Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "eeca4ed1-f94a-4c4b-87d1-158618dbf25d", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-03T20:58:55Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "eeca4ed1-f94a-4c4b-87d1-158618dbf25d", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-03T20:58:55Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "eeca4ed1-f94a-4c4b-87d1-158618dbf25d", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-03T20:58:55Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "eeca4ed1-f94a-4c4b-87d1-158618dbf25d", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm", "name": "premium-simulated-simulated-premium-kserve-route-authn"} 2026-06-03T20:58:55Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "eeca4ed1-f94a-4c4b-87d1-158618dbf25d", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm", "authType": "userdefined"} 2026-06-03T20:58:55Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "eeca4ed1-f94a-4c4b-87d1-158618dbf25d", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-03T20:59:05Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "90c51662-41e6-4503-a301-b68a314a2447", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-03T20:59:05Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "90c51662-41e6-4503-a301-b68a314a2447", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-03T20:59:05Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "90c51662-41e6-4503-a301-b68a314a2447", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-03T20:59:05Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "90c51662-41e6-4503-a301-b68a314a2447", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm", "name": "e2e-distinct-2-simulated-kserve-route-authn"} 2026-06-03T20:59:05Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "90c51662-41e6-4503-a301-b68a314a2447", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-03T20:59:05Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "90c51662-41e6-4503-a301-b68a314a2447", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-03T20:59:06Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-03T20:59:06Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-03T20:59:08Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-03T20:59:08Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-03T20:59:16Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "9f3507e5-6078-43da-81dc-169af773b4bb", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-03T20:59:16Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "9f3507e5-6078-43da-81dc-169af773b4bb", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-03T20:59:16Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "9f3507e5-6078-43da-81dc-169af773b4bb", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-03T20:59:16Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "9f3507e5-6078-43da-81dc-169af773b4bb", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm", "name": "e2e-distinct-simulated-kserve-route-authn"} 2026-06-03T20:59:16Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "9f3507e5-6078-43da-81dc-169af773b4bb", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-03T20:59:16Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "9f3507e5-6078-43da-81dc-169af773b4bb", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-03T20:59:24Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "4ce404c3-9d70-4778-bdef-d5bab1cc206f", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-03T20:59:24Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "4ce404c3-9d70-4778-bdef-d5bab1cc206f", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-03T20:59:24Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "4ce404c3-9d70-4778-bdef-d5bab1cc206f", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-03T20:59:24Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "4ce404c3-9d70-4778-bdef-d5bab1cc206f", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm", "name": "e2e-trlp-test-simulated-kserve-route-authn"} 2026-06-03T20:59:24Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "4ce404c3-9d70-4778-bdef-d5bab1cc206f", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-03T20:59:24Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "4ce404c3-9d70-4778-bdef-d5bab1cc206f", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-03T20:59:28Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "f45bd7b8-e3ea-426d-ae4b-600b0d25bbe8", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:59:28Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "f45bd7b8-e3ea-426d-ae4b-600b0d25bbe8", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:59:28Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "f45bd7b8-e3ea-426d-ae4b-600b0d25bbe8", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:59:28Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "f45bd7b8-e3ea-426d-ae4b-600b0d25bbe8", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated-kserve-route-authn"} 2026-06-03T20:59:28Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "f45bd7b8-e3ea-426d-ae4b-600b0d25bbe8", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-03T20:59:28Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "f45bd7b8-e3ea-426d-ae4b-600b0d25bbe8", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:59:37Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "6e082650-4637-4538-9f2a-81df810b811a", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:59:37Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "6e082650-4637-4538-9f2a-81df810b811a", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:59:37Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "6e082650-4637-4538-9f2a-81df810b811a", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T20:59:37Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "6e082650-4637-4538-9f2a-81df810b811a", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm", "name": "facebook-opt-125m-simulated-kserve-route-authn"} 2026-06-03T20:59:37Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "6e082650-4637-4538-9f2a-81df810b811a", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-03T20:59:37Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "6e082650-4637-4538-9f2a-81df810b811a", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-03T21:00:01Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "efe69c06-f606-4087-aa79-ea1d91282f10", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-03T21:00:01Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "efe69c06-f606-4087-aa79-ea1d91282f10", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-03T21:00:01Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "efe69c06-f606-4087-aa79-ea1d91282f10", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-03T21:00:01Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "efe69c06-f606-4087-aa79-ea1d91282f10", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm", "name": "premium-simulated-simulated-premium-kserve-route-authn"} 2026-06-03T21:00:01Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "efe69c06-f606-4087-aa79-ea1d91282f10", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm", "authType": "userdefined"} 2026-06-03T21:00:01Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "efe69c06-f606-4087-aa79-ea1d91282f10", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-03T21:00:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "openshift-service-ca.crt", "reconcileID": "6853c55b-80e0-4aaf-b892-45af5f3efb9c", "ConfigMap": "openshift-service-ca.crt", "namespace": "premium-users-namespace"} 2026-06-03T21:00:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "odh-trusted-ca-bundle", "reconcileID": "18abc266-80b1-4c73-8dc6-4228d72ca805", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "premium-users-namespace"} 2026-06-03T21:00:05Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "openshift-service-ca.crt", "reconcileID": "e44ad4b1-7374-4bc5-b4ef-98ab61523d7a", "ConfigMap": "openshift-service-ca.crt", "namespace": "premium-users-namespace", "create": "odh-kserve-custom-ca-bundle"} 2026-06-03T21:00:05Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "openshift-service-ca.crt", "reconcileID": "e44ad4b1-7374-4bc5-b4ef-98ab61523d7a", "ConfigMap": "openshift-service-ca.crt", "namespace": "premium-users-namespace"} 2026-06-03T21:00:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "17269295-511f-433d-8716-6e4c47f1122a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "premium-users-namespace"} 2026-06-03T21:00:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "openshift-service-ca.crt", "reconcileID": "6e3bc441-b19c-46a6-85e9-ddcd916497de", "ConfigMap": "openshift-service-ca.crt", "namespace": "premium-users-namespace"} 2026-06-03T21:00:05Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "odh-trusted-ca-bundle", "reconcileID": "e1857434-baa3-4e54-b7eb-c12dc69e302c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "premium-users-namespace", "update": "odh-kserve-custom-ca-bundle"} 2026-06-03T21:00:05Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "odh-trusted-ca-bundle", "reconcileID": "e1857434-baa3-4e54-b7eb-c12dc69e302c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "premium-users-namespace"} 2026-06-03T21:00:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cbb3561a-7e9d-469a-be77-439c8fd6427f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "premium-users-namespace"} 2026-06-03T21:00:06Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "openshift-service-ca.crt", "reconcileID": "d86ea5d4-355a-4636-a8d5-90acb8e123d4", "ConfigMap": "openshift-service-ca.crt", "namespace": "maas-admin"} 2026-06-03T21:00:06Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "odh-trusted-ca-bundle", "reconcileID": "500dabb0-62f0-47d4-8147-92486d711479", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "maas-admin"} 2026-06-03T21:00:06Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "openshift-service-ca.crt", "reconcileID": "0751885e-e044-43df-8882-a8f3915dd141", "ConfigMap": "openshift-service-ca.crt", "namespace": "maas-admin", "create": "odh-kserve-custom-ca-bundle"} 2026-06-03T21:00:06Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "openshift-service-ca.crt", "reconcileID": "0751885e-e044-43df-8882-a8f3915dd141", "ConfigMap": "openshift-service-ca.crt", "namespace": "maas-admin"} 2026-06-03T21:00:06Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "db0a89c7-60dd-43e1-9047-70ac7372e9dd", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "maas-admin"} 2026-06-03T21:00:06Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "openshift-service-ca.crt", "reconcileID": "03625bb9-d6b7-4644-a4cb-b88f616bf9c8", "ConfigMap": "openshift-service-ca.crt", "namespace": "maas-admin"} 2026-06-03T21:00:06Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "odh-trusted-ca-bundle", "reconcileID": "7836f1ef-05bf-4456-84c3-e830c79ea29a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "maas-admin", "update": "odh-kserve-custom-ca-bundle"} 2026-06-03T21:00:06Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "odh-trusted-ca-bundle", "reconcileID": "7836f1ef-05bf-4456-84c3-e830c79ea29a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "maas-admin"} 2026-06-03T21:00:06Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "79cb494d-7a8a-42ac-a4eb-6373a8475edb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "maas-admin"} 2026-06-03T21:02:15Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-trusted-ca-bundle", "reconcileID": "ea5ed694-816f-4a1a-b87b-899af5af5da5", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-api-unwatched-ns"} 2026-06-03T21:02:15Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "7b5a658f-7c23-41be-a64e-f191a05d825c", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-api-unwatched-ns"} 2026-06-03T21:02:15Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "62c34495-83ef-4d7c-a8ad-8b0b42fb90f5", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-api-unwatched-ns", "create": "odh-kserve-custom-ca-bundle"} 2026-06-03T21:02:15Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "62c34495-83ef-4d7c-a8ad-8b0b42fb90f5", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-api-unwatched-ns"} 2026-06-03T21:02:15Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "899ea633-d09f-451d-b649-aa323ebc42ad", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-api-unwatched-ns"} 2026-06-03T21:02:15Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "73d60469-947e-40aa-8957-0494eded41b6", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-api-unwatched-ns"} 2026-06-03T21:02:15Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-trusted-ca-bundle", "reconcileID": "7224c172-88c8-4944-99e9-df9641c89f79", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-api-unwatched-ns", "update": "odh-kserve-custom-ca-bundle"} 2026-06-03T21:02:15Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-trusted-ca-bundle", "reconcileID": "7224c172-88c8-4944-99e9-df9641c89f79", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-api-unwatched-ns"} 2026-06-03T21:02:15Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e6a58c6b-87cb-480c-9a43-8a22a0e185ae", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-api-unwatched-ns"} 2026-06-03T21:02:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5b82ae2a-5a9b-4b11-8823-621673212f80", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-api-unwatched-ns"} 2026-06-03T21:02:29Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5b82ae2a-5a9b-4b11-8823-621673212f80", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-api-unwatched-ns", "create": "odh-kserve-custom-ca-bundle"} 2026-06-03T21:02:29Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5b82ae2a-5a9b-4b11-8823-621673212f80", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-api-unwatched-ns because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-03T21:02:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-trusted-ca-bundle", "reconcileID": "66e670ac-8bc9-4509-9504-479f2350d920", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-api-unwatched-ns"} 2026-06-03T21:02:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-trusted-ca-bundle", "reconcileID": "66e670ac-8bc9-4509-9504-479f2350d920", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-api-unwatched-ns"} 2026-06-03T21:02:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "0f44a339-4eda-416c-9296-1173193d569f", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-api-unwatched-ns"} 2026-06-03T21:02:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "0f44a339-4eda-416c-9296-1173193d569f", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-api-unwatched-ns"} 2026-06-03T21:02:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5d93e8e4-9077-45b8-be33-25cf6e52c62c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-api-unwatched-ns"} 2026-06-03T21:02:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5d93e8e4-9077-45b8-be33-25cf6e52c62c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-api-unwatched-ns"} 2026-06-03T21:03:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "ad7521d2-b4a0-4398-a112-e5a9f4ee8760", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unwatched-ns"} 2026-06-03T21:03:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-trusted-ca-bundle", "reconcileID": "d0917ffb-0c5a-4297-93d1-ece93e524e20", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unwatched-ns"} 2026-06-03T21:03:02Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "a1cbfcea-ace1-48a5-80fb-d8622c6ae48c", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unwatched-ns", "create": "odh-kserve-custom-ca-bundle"} 2026-06-03T21:03:02Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "a1cbfcea-ace1-48a5-80fb-d8622c6ae48c", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unwatched-ns"} 2026-06-03T21:03:02Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "73bbb442-cf7a-491c-8944-251e0a5276aa", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unwatched-ns", "create": "odh-kserve-custom-ca-bundle"} 2026-06-03T21:03:02Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "73bbb442-cf7a-491c-8944-251e0a5276aa", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-03T21:03:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b19cbb43-e2fa-4bac-b05c-89ce31a2cc71", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unwatched-ns"} 2026-06-03T21:03:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "cbab897d-be12-44ee-95a1-5c59fb9a6537", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unwatched-ns"} 2026-06-03T21:03:02Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-trusted-ca-bundle", "reconcileID": "c5fd9de2-e705-4faa-832b-92c050cc17a2", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unwatched-ns", "update": "odh-kserve-custom-ca-bundle"} 2026-06-03T21:03:02Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-trusted-ca-bundle", "reconcileID": "c5fd9de2-e705-4faa-832b-92c050cc17a2", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unwatched-ns"} 2026-06-03T21:03:02Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "815002a7-1958-4460-bd00-3f746103b232", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unwatched-ns"} 2026-06-03T21:03:32Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e1a32065-86d1-4b0a-9f45-972ebb8b1769", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unwatched-ns"} 2026-06-03T21:03:32Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e1a32065-86d1-4b0a-9f45-972ebb8b1769", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unwatched-ns", "create": "odh-kserve-custom-ca-bundle"} 2026-06-03T21:03:32Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e1a32065-86d1-4b0a-9f45-972ebb8b1769", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-unwatched-ns because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-03T21:03:32Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-trusted-ca-bundle", "reconcileID": "03c8afad-ff29-49a5-bc1d-720fa369325c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unwatched-ns"} 2026-06-03T21:03:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-trusted-ca-bundle", "reconcileID": "03c8afad-ff29-49a5-bc1d-720fa369325c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unwatched-ns"} 2026-06-03T21:03:32Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "0005dd2a-7cad-4945-ae4a-2ceeddd5f0f7", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unwatched-ns"} 2026-06-03T21:03:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "0005dd2a-7cad-4945-ae4a-2ceeddd5f0f7", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unwatched-ns"} 2026-06-03T21:03:32Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bf176a2e-7b5f-410a-a3c0-91da7561d49a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unwatched-ns"} 2026-06-03T21:03:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bf176a2e-7b5f-410a-a3c0-91da7561d49a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unwatched-ns"} 2026-06-03T21:03:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-8354b6"}, "namespace": "e2e-modelref-8354b6", "name": "openshift-service-ca.crt", "reconcileID": "77225b81-74ca-4249-b6ed-139e92b56e32", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-8354b6"} 2026-06-03T21:03:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-modelref-8354b6"}, "namespace": "e2e-modelref-8354b6", "name": "odh-trusted-ca-bundle", "reconcileID": "1d360193-7537-4cf7-8881-d8bc17e4b0d9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-modelref-8354b6"} 2026-06-03T21:03:32Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-8354b6"}, "namespace": "e2e-modelref-8354b6", "name": "openshift-service-ca.crt", "reconcileID": "d3e41452-8762-48e9-b12d-6b62154fb3ff", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-8354b6", "create": "odh-kserve-custom-ca-bundle"} 2026-06-03T21:03:32Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-8354b6"}, "namespace": "e2e-modelref-8354b6", "name": "openshift-service-ca.crt", "reconcileID": "d3e41452-8762-48e9-b12d-6b62154fb3ff", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-8354b6"} 2026-06-03T21:03:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-8354b6"}, "namespace": "e2e-modelref-8354b6", "name": "openshift-service-ca.crt", "reconcileID": "13ec645e-a450-4ef4-ab81-9c32d8b8079d", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-8354b6"} 2026-06-03T21:03:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-8354b6"}, "namespace": "e2e-modelref-8354b6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "319adbfe-1f95-4fa0-a065-f14d72ee37bb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-8354b6"} 2026-06-03T21:03:32Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-modelref-8354b6"}, "namespace": "e2e-modelref-8354b6", "name": "odh-trusted-ca-bundle", "reconcileID": "7548b00d-8cea-4206-8eff-ea786aa157fa", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-modelref-8354b6", "update": "odh-kserve-custom-ca-bundle"} 2026-06-03T21:03:32Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-modelref-8354b6"}, "namespace": "e2e-modelref-8354b6", "name": "odh-trusted-ca-bundle", "reconcileID": "7548b00d-8cea-4206-8eff-ea786aa157fa", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-modelref-8354b6"} 2026-06-03T21:03:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-8354b6"}, "namespace": "e2e-modelref-8354b6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5962827a-ab22-404b-b700-d2eb5238b4d0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-8354b6"} 2026-06-03T21:03:33Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "4fd6401f-bc5b-4cfb-b98f-6935d94f2d9d", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:03:33Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "4fd6401f-bc5b-4cfb-b98f-6935d94f2d9d", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T21:03:33Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "4fd6401f-bc5b-4cfb-b98f-6935d94f2d9d", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:03:33Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "a4e81c5b-e8e9-43db-ae76-106d5f7b5b65", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:03:33Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "a4e81c5b-e8e9-43db-ae76-106d5f7b5b65", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T21:03:33Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "a4e81c5b-e8e9-43db-ae76-106d5f7b5b65", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:03:49Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "d49e1aa6-e1ef-44d1-8ae2-9555a182c6f4", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:03:49Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "d49e1aa6-e1ef-44d1-8ae2-9555a182c6f4", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T21:03:49Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "d49e1aa6-e1ef-44d1-8ae2-9555a182c6f4", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:03:49Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "23c03eaf-0050-42b7-91de-28c011ccb100", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:03:49Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "23c03eaf-0050-42b7-91de-28c011ccb100", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T21:03:49Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "23c03eaf-0050-42b7-91de-28c011ccb100", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:03:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-8354b6"}, "namespace": "e2e-modelref-8354b6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2593270a-7a9e-48fd-bb48-c0b6708b7598", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-8354b6"} 2026-06-03T21:03:55Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-8354b6"}, "namespace": "e2e-modelref-8354b6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2593270a-7a9e-48fd-bb48-c0b6708b7598", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-8354b6", "create": "odh-kserve-custom-ca-bundle"} 2026-06-03T21:03:55Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-8354b6"}, "namespace": "e2e-modelref-8354b6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2593270a-7a9e-48fd-bb48-c0b6708b7598", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-modelref-8354b6 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-03T21:03:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-modelref-8354b6"}, "namespace": "e2e-modelref-8354b6", "name": "odh-trusted-ca-bundle", "reconcileID": "baf97c72-0571-4b1b-b3e1-9b3ff03611ee", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-modelref-8354b6"} 2026-06-03T21:03:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-modelref-8354b6"}, "namespace": "e2e-modelref-8354b6", "name": "odh-trusted-ca-bundle", "reconcileID": "baf97c72-0571-4b1b-b3e1-9b3ff03611ee", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-modelref-8354b6"} 2026-06-03T21:03:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-8354b6"}, "namespace": "e2e-modelref-8354b6", "name": "openshift-service-ca.crt", "reconcileID": "834db75b-ca8b-4adc-ad8a-87829170b91f", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-8354b6"} 2026-06-03T21:03:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-8354b6"}, "namespace": "e2e-modelref-8354b6", "name": "openshift-service-ca.crt", "reconcileID": "834db75b-ca8b-4adc-ad8a-87829170b91f", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-8354b6"} 2026-06-03T21:03:55Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-8354b6"}, "namespace": "e2e-modelref-8354b6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "71d9b5b0-ec31-4b2b-885c-52c7ac5c3914", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-8354b6"} 2026-06-03T21:03:55Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-8354b6"}, "namespace": "e2e-modelref-8354b6", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "71d9b5b0-ec31-4b2b-885c-52c7ac5c3914", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-8354b6"} 2026-06-03T21:04:03Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-f83385"}, "namespace": "e2e-modelref-f83385", "name": "openshift-service-ca.crt", "reconcileID": "655f2fee-4f3f-459b-814e-767842947897", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-f83385"} 2026-06-03T21:04:03Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-modelref-f83385"}, "namespace": "e2e-modelref-f83385", "name": "odh-trusted-ca-bundle", "reconcileID": "b554e67f-819d-4eaf-83f3-e23190c06e9f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-modelref-f83385"} 2026-06-03T21:04:03Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-f83385"}, "namespace": "e2e-modelref-f83385", "name": "openshift-service-ca.crt", "reconcileID": "a23e9c4b-828f-46f5-88a2-9c1a03a1ca9d", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-f83385", "create": "odh-kserve-custom-ca-bundle"} 2026-06-03T21:04:03Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-f83385"}, "namespace": "e2e-modelref-f83385", "name": "openshift-service-ca.crt", "reconcileID": "a23e9c4b-828f-46f5-88a2-9c1a03a1ca9d", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-f83385"} 2026-06-03T21:04:03Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-f83385"}, "namespace": "e2e-modelref-f83385", "name": "openshift-service-ca.crt", "reconcileID": "ea5b511f-2bca-4351-9568-d0693e6f4972", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-f83385"} 2026-06-03T21:04:03Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-f83385"}, "namespace": "e2e-modelref-f83385", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bca05459-198e-47ae-8f0e-60881ed74512", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-f83385"} 2026-06-03T21:04:03Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "f5274e38-1335-4308-843b-3fca3729683e", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:04:03Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "f5274e38-1335-4308-843b-3fca3729683e", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T21:04:04Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "f5274e38-1335-4308-843b-3fca3729683e", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:04:04Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-modelref-f83385"}, "namespace": "e2e-modelref-f83385", "name": "odh-trusted-ca-bundle", "reconcileID": "4f1ff8f8-6e1a-43fc-84f4-7877c8843e3d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-modelref-f83385", "update": "odh-kserve-custom-ca-bundle"} 2026-06-03T21:04:04Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "b563ab31-6d7f-4ab1-b688-8a0021592ac3", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:04:04Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "b563ab31-6d7f-4ab1-b688-8a0021592ac3", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T21:04:04Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "b563ab31-6d7f-4ab1-b688-8a0021592ac3", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:04:04Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-modelref-f83385"}, "namespace": "e2e-modelref-f83385", "name": "odh-trusted-ca-bundle", "reconcileID": "4f1ff8f8-6e1a-43fc-84f4-7877c8843e3d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-modelref-f83385"} 2026-06-03T21:04:04Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-f83385"}, "namespace": "e2e-modelref-f83385", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "65a28570-d366-4626-9c3f-20a0a1cd8e05", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-f83385"} 2026-06-03T21:04:20Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "8ced4776-4e8d-489f-8b23-f09682336c91", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:04:20Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "8ced4776-4e8d-489f-8b23-f09682336c91", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T21:04:20Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "8ced4776-4e8d-489f-8b23-f09682336c91", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:04:20Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "e1f78216-7312-40c9-ad0a-061ba7b07744", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:04:20Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "e1f78216-7312-40c9-ad0a-061ba7b07744", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T21:04:20Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "e1f78216-7312-40c9-ad0a-061ba7b07744", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:04:25Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-f83385"}, "namespace": "e2e-modelref-f83385", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "05b221de-d9a3-4e55-aecd-96670c78df11", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-f83385"} 2026-06-03T21:04:25Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-f83385"}, "namespace": "e2e-modelref-f83385", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "05b221de-d9a3-4e55-aecd-96670c78df11", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-f83385", "create": "odh-kserve-custom-ca-bundle"} 2026-06-03T21:04:25Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-f83385"}, "namespace": "e2e-modelref-f83385", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "05b221de-d9a3-4e55-aecd-96670c78df11", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-modelref-f83385 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-03T21:04:25Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-modelref-f83385"}, "namespace": "e2e-modelref-f83385", "name": "odh-trusted-ca-bundle", "reconcileID": "74de10ee-54aa-4c1c-8217-371b15ceb66e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-modelref-f83385"} 2026-06-03T21:04:25Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-modelref-f83385"}, "namespace": "e2e-modelref-f83385", "name": "odh-trusted-ca-bundle", "reconcileID": "74de10ee-54aa-4c1c-8217-371b15ceb66e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-modelref-f83385"} 2026-06-03T21:04:25Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-f83385"}, "namespace": "e2e-modelref-f83385", "name": "openshift-service-ca.crt", "reconcileID": "dd0f7b80-4919-4373-8f7f-ac91e804ab54", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-f83385"} 2026-06-03T21:04:25Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-f83385"}, "namespace": "e2e-modelref-f83385", "name": "openshift-service-ca.crt", "reconcileID": "dd0f7b80-4919-4373-8f7f-ac91e804ab54", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-f83385"} 2026-06-03T21:04:25Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-f83385"}, "namespace": "e2e-modelref-f83385", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "89584135-0fb0-485e-88c3-e4172389b7af", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-f83385"} 2026-06-03T21:04:25Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-f83385"}, "namespace": "e2e-modelref-f83385", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "89584135-0fb0-485e-88c3-e4172389b7af", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-f83385"} 2026-06-03T21:16:30Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-03T21:16:30Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-03T21:16:34Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-03T21:16:34Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-03T21:22:15Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "b6c63228-2bd6-47cf-9e5b-5fd9f2a9729c", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:22:15Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "b6c63228-2bd6-47cf-9e5b-5fd9f2a9729c", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T21:22:15Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "b6c63228-2bd6-47cf-9e5b-5fd9f2a9729c", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:22:25Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "e5ce6cd2-139d-412a-bff6-b5ff181af2cc", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:22:25Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "e5ce6cd2-139d-412a-bff6-b5ff181af2cc", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T21:22:25Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "e5ce6cd2-139d-412a-bff6-b5ff181af2cc", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:22:49Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "2ce585c9-a4de-4f74-87b5-e090fc5ac505", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:22:49Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "2ce585c9-a4de-4f74-87b5-e090fc5ac505", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T21:22:49Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "2ce585c9-a4de-4f74-87b5-e090fc5ac505", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:23:02Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "f4261141-f3ab-46f3-99a7-940fa54fff10", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:23:02Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "f4261141-f3ab-46f3-99a7-940fa54fff10", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-03T21:23:02Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "f4261141-f3ab-46f3-99a7-940fa54fff10", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-03T21:23:04Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-3bc3a4ce"}, "namespace": "e2e-aitenant-3bc3a4ce", "name": "openshift-service-ca.crt", "reconcileID": "e3780954-47d8-4e6e-8e64-1276413e771a", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-3bc3a4ce"} 2026-06-03T21:23:04Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-3bc3a4ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-3bc3a4ce", "reconcileID": "eed93676-637a-432b-8c3f-40f4263ebd02", "Gateway": "e2e-ait-3bc3a4ce", "namespace": "openshift-ingress"} 2026-06-03T21:23:04Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-3bc3a4ce"}, "namespace": "e2e-aitenant-3bc3a4ce", "name": "odh-trusted-ca-bundle", "reconcileID": "c46bf3da-5bac-48f6-81c9-4de6d1515169", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-3bc3a4ce"} 2026-06-03T21:23:04Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-3bc3a4ce"}, "namespace": "e2e-aitenant-3bc3a4ce", "name": "openshift-service-ca.crt", "reconcileID": "c3981dd6-69fc-4339-87bb-fa6d85fd857e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-3bc3a4ce", "create": "odh-kserve-custom-ca-bundle"} 2026-06-03T21:23:04Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-3bc3a4ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-3bc3a4ce", "reconcileID": "828e267c-5dc1-4304-a4ae-d5e9b0720135", "Gateway": "e2e-ait-3bc3a4ce", "namespace": "openshift-ingress"} 2026-06-03T21:23:04Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-3bc3a4ce"}, "namespace": "e2e-aitenant-3bc3a4ce", "name": "openshift-service-ca.crt", "reconcileID": "c3981dd6-69fc-4339-87bb-fa6d85fd857e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-3bc3a4ce"} 2026-06-03T21:23:04Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-3bc3a4ce"}, "namespace": "e2e-aitenant-3bc3a4ce", "name": "openshift-service-ca.crt", "reconcileID": "acd5c2b6-a339-443f-b921-a613089d31dd", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-3bc3a4ce"} 2026-06-03T21:23:04Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-3bc3a4ce"}, "namespace": "e2e-aitenant-3bc3a4ce", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0e65a240-98aa-4db2-928d-3439333fb8d0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-3bc3a4ce"} 2026-06-03T21:23:04Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-3bc3a4ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-3bc3a4ce", "reconcileID": "c249c333-8af6-4550-aa69-c76b781c37c7", "Gateway": "e2e-ait-3bc3a4ce", "namespace": "openshift-ingress"} 2026-06-03T21:23:04Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-3bc3a4ce"}, "namespace": "e2e-aitenant-3bc3a4ce", "name": "odh-trusted-ca-bundle", "reconcileID": "e7ac9645-1f5b-4827-8a55-d430344cc8cb", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-3bc3a4ce", "update": "odh-kserve-custom-ca-bundle"} 2026-06-03T21:23:04Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-3bc3a4ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-3bc3a4ce", "reconcileID": "c1684e2a-6c5a-4e8b-8690-661ffb1e6d8d", "Gateway": "e2e-ait-3bc3a4ce", "namespace": "openshift-ingress"} 2026-06-03T21:23:04Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-3bc3a4ce"}, "namespace": "e2e-aitenant-3bc3a4ce", "name": "odh-trusted-ca-bundle", "reconcileID": "e7ac9645-1f5b-4827-8a55-d430344cc8cb", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-3bc3a4ce"} 2026-06-03T21:23:04Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-3bc3a4ce"}, "namespace": "e2e-aitenant-3bc3a4ce", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "178f7a67-1915-43f9-b727-6c84392428a1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-3bc3a4ce"} 2026-06-03T21:23:04Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-3bc3a4ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-3bc3a4ce", "reconcileID": "2f2cb349-8ba0-4de7-ba6a-df7e25a994e8", "Gateway": "e2e-ait-3bc3a4ce", "namespace": "openshift-ingress"} 2026-06-03T21:23:08Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-3bc3a4ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-3bc3a4ce", "reconcileID": "e16d41a2-db63-4e5a-84b8-b8eceb44d84c", "Gateway": "e2e-ait-3bc3a4ce", "namespace": "openshift-ingress"} 2026-06-03T21:23:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-3bc3a4ce"}, "namespace": "e2e-aitenant-3bc3a4ce", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "76754775-fb8f-4e87-994f-11d85466abc6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-3bc3a4ce"} 2026-06-03T21:23:16Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-3bc3a4ce"}, "namespace": "e2e-aitenant-3bc3a4ce", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "76754775-fb8f-4e87-994f-11d85466abc6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-3bc3a4ce", "create": "odh-kserve-custom-ca-bundle"} 2026-06-03T21:23:16Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-3bc3a4ce"}, "namespace": "e2e-aitenant-3bc3a4ce", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "76754775-fb8f-4e87-994f-11d85466abc6", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-aitenant-3bc3a4ce because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-03T21:23:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-3bc3a4ce"}, "namespace": "e2e-aitenant-3bc3a4ce", "name": "odh-trusted-ca-bundle", "reconcileID": "fd2db83f-4e02-4dff-836d-bf8631749eba", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-3bc3a4ce"} 2026-06-03T21:23:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-3bc3a4ce"}, "namespace": "e2e-aitenant-3bc3a4ce", "name": "odh-trusted-ca-bundle", "reconcileID": "fd2db83f-4e02-4dff-836d-bf8631749eba", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-3bc3a4ce"} 2026-06-03T21:23:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-3bc3a4ce"}, "namespace": "e2e-aitenant-3bc3a4ce", "name": "openshift-service-ca.crt", "reconcileID": "2ba14182-13b8-40bc-b0cb-c83924b1fa01", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-3bc3a4ce"} 2026-06-03T21:23:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-3bc3a4ce"}, "namespace": "e2e-aitenant-3bc3a4ce", "name": "openshift-service-ca.crt", "reconcileID": "2ba14182-13b8-40bc-b0cb-c83924b1fa01", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-3bc3a4ce"} 2026-06-03T21:23:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-3bc3a4ce"}, "namespace": "e2e-aitenant-3bc3a4ce", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "32f89cf0-afdf-449c-8b9a-501a46d840c2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-3bc3a4ce"} 2026-06-03T21:23:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-3bc3a4ce"}, "namespace": "e2e-aitenant-3bc3a4ce", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "32f89cf0-afdf-449c-8b9a-501a46d840c2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-3bc3a4ce"} 2026-06-03T21:23:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-4eb444c2"}, "namespace": "e2e-aitenant-4eb444c2", "name": "openshift-service-ca.crt", "reconcileID": "d0782a55-689b-4dcb-981a-f315bd702751", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-4eb444c2"} 2026-06-03T21:23:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-4eb444c2"}, "namespace": "e2e-aitenant-4eb444c2", "name": "odh-trusted-ca-bundle", "reconcileID": "73814440-d51d-43cc-abb3-bfa31eb95f45", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-4eb444c2"} 2026-06-03T21:23:16Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-4eb444c2"}, "namespace": "e2e-aitenant-4eb444c2", "name": "openshift-service-ca.crt", "reconcileID": "4f4da7d0-bbcc-4a9c-925d-614f1ef20760", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-4eb444c2", "create": "odh-kserve-custom-ca-bundle"} 2026-06-03T21:23:16Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-4eb444c2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-4eb444c2", "reconcileID": "e35e798e-36f2-48d7-80f1-195c965c998b", "Gateway": "e2e-ait-4eb444c2", "namespace": "openshift-ingress"} 2026-06-03T21:23:16Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-4eb444c2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-4eb444c2", "reconcileID": "c557bae1-4b5c-4b6e-a5da-52f05927b38e", "Gateway": "e2e-ait-4eb444c2", "namespace": "openshift-ingress"} 2026-06-03T21:23:16Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-4eb444c2"}, "namespace": "e2e-aitenant-4eb444c2", "name": "openshift-service-ca.crt", "reconcileID": "4f4da7d0-bbcc-4a9c-925d-614f1ef20760", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-4eb444c2"} 2026-06-03T21:23:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-4eb444c2"}, "namespace": "e2e-aitenant-4eb444c2", "name": "openshift-service-ca.crt", "reconcileID": "14a253be-a3d1-4605-ae56-2e6efa00b3f0", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-4eb444c2"} 2026-06-03T21:23:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-4eb444c2"}, "namespace": "e2e-aitenant-4eb444c2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ac2a0c1e-7819-4679-870c-a83525f8baa4", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-4eb444c2"} 2026-06-03T21:23:16Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-4eb444c2"}, "namespace": "e2e-aitenant-4eb444c2", "name": "odh-trusted-ca-bundle", "reconcileID": "8f4eb714-7dd8-4531-a679-0de32751bf9a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-4eb444c2", "update": "odh-kserve-custom-ca-bundle"} 2026-06-03T21:23:16Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-4eb444c2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-4eb444c2", "reconcileID": "65db2965-c6d3-4075-af7d-9b86addf7bcb", "Gateway": "e2e-ait-4eb444c2", "namespace": "openshift-ingress"} 2026-06-03T21:23:16Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-4eb444c2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-4eb444c2", "reconcileID": "58fe1e6b-f5e1-4f39-bf7d-75f20d763f12", "Gateway": "e2e-ait-4eb444c2", "namespace": "openshift-ingress"} 2026-06-03T21:23:16Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-4eb444c2"}, "namespace": "e2e-aitenant-4eb444c2", "name": "odh-trusted-ca-bundle", "reconcileID": "8f4eb714-7dd8-4531-a679-0de32751bf9a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-4eb444c2"} 2026-06-03T21:23:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-4eb444c2"}, "namespace": "e2e-aitenant-4eb444c2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bcce46d9-5c9c-4146-a4a6-69370c5c631e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-4eb444c2"} 2026-06-03T21:23:17Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-4eb444c2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-4eb444c2", "reconcileID": "c03afe11-1a58-4def-8a52-a5434984bca1", "Gateway": "e2e-ait-4eb444c2", "namespace": "openshift-ingress"} 2026-06-03T21:23:18Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-4eb444c2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-4eb444c2", "reconcileID": "327472a3-fb4c-4413-b932-e9d8c7432f63", "Gateway": "e2e-ait-4eb444c2", "namespace": "openshift-ingress"} 2026-06-03T21:23:20Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-4eb444c2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-4eb444c2", "reconcileID": "a8259ce9-1004-4971-a8b1-3c98b07b21a3", "Gateway": "e2e-ait-4eb444c2", "namespace": "openshift-ingress"} 2026-06-03T21:23:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-4eb444c2"}, "namespace": "e2e-aitenant-4eb444c2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "81c5ce8b-eecd-4ce6-b3a3-9531f47a1dbd", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-4eb444c2"} 2026-06-03T21:23:29Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-4eb444c2"}, "namespace": "e2e-aitenant-4eb444c2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "81c5ce8b-eecd-4ce6-b3a3-9531f47a1dbd", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-4eb444c2", "create": "odh-kserve-custom-ca-bundle"} 2026-06-03T21:23:29Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-4eb444c2"}, "namespace": "e2e-aitenant-4eb444c2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "81c5ce8b-eecd-4ce6-b3a3-9531f47a1dbd", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-aitenant-4eb444c2 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-03T21:23:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-4eb444c2"}, "namespace": "e2e-aitenant-4eb444c2", "name": "odh-trusted-ca-bundle", "reconcileID": "0c313a03-7ff1-4346-90d9-aef09cc1a20d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-4eb444c2"} 2026-06-03T21:23:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-4eb444c2"}, "namespace": "e2e-aitenant-4eb444c2", "name": "odh-trusted-ca-bundle", "reconcileID": "0c313a03-7ff1-4346-90d9-aef09cc1a20d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-4eb444c2"} 2026-06-03T21:23:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-4eb444c2"}, "namespace": "e2e-aitenant-4eb444c2", "name": "openshift-service-ca.crt", "reconcileID": "f86b200d-6003-423a-a254-376d0118b1b2", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-4eb444c2"} 2026-06-03T21:23:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-4eb444c2"}, "namespace": "e2e-aitenant-4eb444c2", "name": "openshift-service-ca.crt", "reconcileID": "f86b200d-6003-423a-a254-376d0118b1b2", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-4eb444c2"} 2026-06-03T21:23:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-4eb444c2"}, "namespace": "e2e-aitenant-4eb444c2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a06072a5-16af-4b60-836e-14522af84ba9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-4eb444c2"} 2026-06-03T21:23:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-4eb444c2"}, "namespace": "e2e-aitenant-4eb444c2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a06072a5-16af-4b60-836e-14522af84ba9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-4eb444c2"}