/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-10T22:54:50Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-3ca82a43"}, "namespace": "e2e-webhook-auth-3ca82a43", "name": "odh-trusted-ca-bundle", "reconcileID": "5e2d2a75-2006-4163-a0e3-1adb785fb529", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-3ca82a43"} 2026-07-10T22:54:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-webhook-auth-3ca82a43"}, "namespace": "e2e-webhook-auth-3ca82a43", "name": "odh-trusted-ca-bundle", "reconcileID": "5e2d2a75-2006-4163-a0e3-1adb785fb529", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-webhook-auth-3ca82a43"} 2026-07-10T22:54:50Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-3ca82a43"}, "namespace": "e2e-webhook-auth-3ca82a43", "name": "openshift-service-ca.crt", "reconcileID": "6ffae656-f041-4dd0-8511-fc3875c356c1", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-3ca82a43"} 2026-07-10T22:54:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-webhook-auth-3ca82a43"}, "namespace": "e2e-webhook-auth-3ca82a43", "name": "openshift-service-ca.crt", "reconcileID": "6ffae656-f041-4dd0-8511-fc3875c356c1", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-webhook-auth-3ca82a43"} 2026-07-10T22:54:50Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-3ca82a43"}, "namespace": "e2e-webhook-auth-3ca82a43", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "71109673-2066-4553-a452-037854c07fd0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-3ca82a43"} 2026-07-10T22:54:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-webhook-auth-3ca82a43"}, "namespace": "e2e-webhook-auth-3ca82a43", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "71109673-2066-4553-a452-037854c07fd0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-webhook-auth-3ca82a43"} 2026-07-10T22:54:50Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-10T22:54:50Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-10T22:54:53Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-10T22:54:53Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-10T22:54:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "2ff8509e-24f5-41a4-9b9c-1e32e1ff4882", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:54:57Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "2ff8509e-24f5-41a4-9b9c-1e32e1ff4882", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress", "name": "e2e-mt-727180d9-authn-ssl"} 2026-07-10T22:54:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "2ff8509e-24f5-41a4-9b9c-1e32e1ff4882", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:54:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "6dacbfc9-85dd-4f88-827e-d44d63af20e7", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:54:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "6dacbfc9-85dd-4f88-827e-d44d63af20e7", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress", "name": "e2e-mt-727180d9-authn-ssl"} 2026-07-10T22:54:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "6dacbfc9-85dd-4f88-827e-d44d63af20e7", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:54:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "0fd0ba8f-4ee8-428e-beb7-a097988f2c9a", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:54:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "0fd0ba8f-4ee8-428e-beb7-a097988f2c9a", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress", "name": "e2e-mt-727180d9-authn-ssl"} 2026-07-10T22:54:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "0fd0ba8f-4ee8-428e-beb7-a097988f2c9a", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:54:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "b2d4603a-f120-4616-ba96-0575cf722041", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:54:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "b2d4603a-f120-4616-ba96-0575cf722041", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress", "name": "e2e-mt-727180d9-authn-ssl"} 2026-07-10T22:54:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "b2d4603a-f120-4616-ba96-0575cf722041", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:54:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "25068a35-52db-472c-a15d-78c1aa45ba19", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:54:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "25068a35-52db-472c-a15d-78c1aa45ba19", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress", "name": "e2e-mt-727180d9-authn-ssl"} 2026-07-10T22:54:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "25068a35-52db-472c-a15d-78c1aa45ba19", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:54:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "1f653c21-dc49-40ea-96db-1586428d8127", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:54:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "1f653c21-dc49-40ea-96db-1586428d8127", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress", "name": "e2e-mt-727180d9-authn-ssl"} 2026-07-10T22:54:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "1f653c21-dc49-40ea-96db-1586428d8127", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:54:58Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "dd0c63ac-2bc3-4ec9-9b12-59b023e83f01", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:54:58Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "dd0c63ac-2bc3-4ec9-9b12-59b023e83f01", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress", "name": "e2e-mt-727180d9-authn-ssl"} 2026-07-10T22:54:58Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "dd0c63ac-2bc3-4ec9-9b12-59b023e83f01", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:55:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "openshift-service-ca.crt", "reconcileID": "6b843f59-0b67-4524-a9bc-ab6e97d13ec1", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:55:00Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "openshift-service-ca.crt", "reconcileID": "1faabb6a-db81-4540-82f1-be36b176c771", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-727180d9", "create": "odh-kserve-custom-ca-bundle"} 2026-07-10T22:55:00Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "openshift-service-ca.crt", "reconcileID": "1faabb6a-db81-4540-82f1-be36b176c771", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:55:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "odh-trusted-ca-bundle", "reconcileID": "ae5eda85-04c5-42cb-8b38-3dbd6c9a62df", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:55:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "aed8c5ec-6380-456c-b441-0c78c28d9fd0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:55:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "openshift-service-ca.crt", "reconcileID": "be8dfa66-13cd-43b2-9981-cdfae17ce8be", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:55:00Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "odh-trusted-ca-bundle", "reconcileID": "9505ab5e-95f1-4806-b548-79b7ba345934", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-727180d9", "update": "odh-kserve-custom-ca-bundle"} 2026-07-10T22:55:00Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "odh-trusted-ca-bundle", "reconcileID": "9505ab5e-95f1-4806-b548-79b7ba345934", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:55:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c393817e-5fd7-43da-87d1-cb3249d0847c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:55:01Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "dc7d376c-c8f1-4dd5-97e4-777ec7c8c2b4", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:55:01Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "dc7d376c-c8f1-4dd5-97e4-777ec7c8c2b4", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress", "name": "e2e-mt-727180d9-authn-ssl"} 2026-07-10T22:55:01Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "dc7d376c-c8f1-4dd5-97e4-777ec7c8c2b4", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:55:01Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "c8d33451-eeca-4a24-b595-e2e2ea7a0fcf", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-10T22:55:05Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "a6cb592a-bb54-4247-8f67-97beca57650f", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:55:05Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "a6cb592a-bb54-4247-8f67-97beca57650f", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:55:05Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "a6cb592a-bb54-4247-8f67-97beca57650f", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:55:05Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "7ec50927-e1e2-4df0-ad0a-613b77699088", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:55:05Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "a6cb592a-bb54-4247-8f67-97beca57650f", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9-kserve-route-authn"} 2026-07-10T22:55:05Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "a6cb592a-bb54-4247-8f67-97beca57650f", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9", "authType": "userdefined"} 2026-07-10T22:55:05Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "a6cb592a-bb54-4247-8f67-97beca57650f", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:55:05Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "7ec50927-e1e2-4df0-ad0a-613b77699088", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress", "name": "e2e-mt-727180d9-authn-ssl"} 2026-07-10T22:55:05Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "7ec50927-e1e2-4df0-ad0a-613b77699088", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:55:05Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "5a82bea1-cf34-4097-aa24-0eeedc63257a", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:55:05Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "5a82bea1-cf34-4097-aa24-0eeedc63257a", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:55:05Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "5a82bea1-cf34-4097-aa24-0eeedc63257a", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:55:05Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "5a82bea1-cf34-4097-aa24-0eeedc63257a", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9-kserve-route-authn"} 2026-07-10T22:55:05Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "5a82bea1-cf34-4097-aa24-0eeedc63257a", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9", "authType": "userdefined"} 2026-07-10T22:55:05Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "5a82bea1-cf34-4097-aa24-0eeedc63257a", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:55:07Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "326e16ba-c62b-4542-ad9a-ee665d9f408e", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:55:07Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "326e16ba-c62b-4542-ad9a-ee665d9f408e", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress", "name": "e2e-mt-727180d9-authn-ssl"} 2026-07-10T22:55:07Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "326e16ba-c62b-4542-ad9a-ee665d9f408e", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:55:36Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "545f06c1-0f01-47f7-b6ed-edb1a676f0b9", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:55:36Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "545f06c1-0f01-47f7-b6ed-edb1a676f0b9", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress", "name": "e2e-mt-727180d9-authn-ssl"} 2026-07-10T22:55:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "545f06c1-0f01-47f7-b6ed-edb1a676f0b9", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:55:37Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "0e428b3f-4e3f-4306-9b14-e7eb8b749cd6", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:55:37Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "0e428b3f-4e3f-4306-9b14-e7eb8b749cd6", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:55:37Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "0e428b3f-4e3f-4306-9b14-e7eb8b749cd6", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:55:37Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "0e428b3f-4e3f-4306-9b14-e7eb8b749cd6", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9-kserve-route-authn"} 2026-07-10T22:55:37Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "0e428b3f-4e3f-4306-9b14-e7eb8b749cd6", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9", "authType": "userdefined"} 2026-07-10T22:55:37Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "0e428b3f-4e3f-4306-9b14-e7eb8b749cd6", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:55:50Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "71cbec4a-8847-4eee-a0de-a281569558ae", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:55:50Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "71cbec4a-8847-4eee-a0de-a281569558ae", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:55:50Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "71cbec4a-8847-4eee-a0de-a281569558ae", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:55:50Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "71cbec4a-8847-4eee-a0de-a281569558ae", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9-kserve-route-authn"} 2026-07-10T22:55:50Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "71cbec4a-8847-4eee-a0de-a281569558ae", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9", "authType": "userdefined"} 2026-07-10T22:55:50Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "71cbec4a-8847-4eee-a0de-a281569558ae", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:55:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "7b01b317-7c99-4093-a6db-df1d06835fe0", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:55:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "7b01b317-7c99-4093-a6db-df1d06835fe0", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress", "name": "e2e-mt-727180d9-authn-ssl"} 2026-07-10T22:55:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "7b01b317-7c99-4093-a6db-df1d06835fe0", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:56:02Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "2f2b8020-305f-4b27-ade8-a53ebe783d63", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:56:02Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "2f2b8020-305f-4b27-ade8-a53ebe783d63", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress", "name": "e2e-mt-727180d9-authn-ssl"} 2026-07-10T22:56:02Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "2f2b8020-305f-4b27-ade8-a53ebe783d63", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:56:02Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "3e36b057-c06a-4acb-b3e6-8c0a9bcff766", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:56:02Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "3e36b057-c06a-4acb-b3e6-8c0a9bcff766", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress", "name": "e2e-mt-727180d9-authn-ssl"} 2026-07-10T22:56:02Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "3e36b057-c06a-4acb-b3e6-8c0a9bcff766", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:56:09Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "018f6f13-1262-446b-9e3b-1cb588d57a5e", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:09Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "018f6f13-1262-446b-9e3b-1cb588d57a5e", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:09Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "018f6f13-1262-446b-9e3b-1cb588d57a5e", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:09Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "018f6f13-1262-446b-9e3b-1cb588d57a5e", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9-kserve-route-authn"} 2026-07-10T22:56:09Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "018f6f13-1262-446b-9e3b-1cb588d57a5e", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9", "authType": "userdefined"} 2026-07-10T22:56:09Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "018f6f13-1262-446b-9e3b-1cb588d57a5e", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:45Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "58e083ec-5349-4726-b98e-887052fab3e0", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:45Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "58e083ec-5349-4726-b98e-887052fab3e0", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:45Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "58e083ec-5349-4726-b98e-887052fab3e0", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:45Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "58e083ec-5349-4726-b98e-887052fab3e0", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9-kserve-route-authn"} 2026-07-10T22:56:45Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "58e083ec-5349-4726-b98e-887052fab3e0", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9", "authType": "userdefined"} 2026-07-10T22:56:45Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "58e083ec-5349-4726-b98e-887052fab3e0", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:45Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "130b00d0-6ae1-44fc-86e1-a163e2f4808d", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:56:45Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "130b00d0-6ae1-44fc-86e1-a163e2f4808d", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress", "name": "e2e-mt-727180d9-authn-ssl"} 2026-07-10T22:56:45Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "130b00d0-6ae1-44fc-86e1-a163e2f4808d", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:56:45Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "f779f8bb-f226-4216-9716-587ed92cad20", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:45Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "f779f8bb-f226-4216-9716-587ed92cad20", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9"} 2026-07-10T22:56:45Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "f779f8bb-f226-4216-9716-587ed92cad20", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:45Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "f779f8bb-f226-4216-9716-587ed92cad20", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:45Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "f779f8bb-f226-4216-9716-587ed92cad20", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9-kserve-route-authn"} 2026-07-10T22:56:45Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "f779f8bb-f226-4216-9716-587ed92cad20", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9-kserve-route-authn"} 2026-07-10T22:56:45Z DEBUG Triggering LLMInferenceService Cleanup for Namespace {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "f779f8bb-f226-4216-9716-587ed92cad20", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:45Z DEBUG Cleaning up LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "f779f8bb-f226-4216-9716-587ed92cad20", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:45Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2a0babf5-f9ab-4060-8da0-621dfa2351e8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:45Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2a0babf5-f9ab-4060-8da0-621dfa2351e8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-727180d9", "create": "odh-kserve-custom-ca-bundle"} 2026-07-10T22:56:45Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2a0babf5-f9ab-4060-8da0-621dfa2351e8", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-727180d9 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-10T22:56:45Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "odh-trusted-ca-bundle", "reconcileID": "8b14e33b-e9aa-443b-ad0f-b3991b78c84f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:45Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "odh-trusted-ca-bundle", "reconcileID": "8b14e33b-e9aa-443b-ad0f-b3991b78c84f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:45Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "openshift-service-ca.crt", "reconcileID": "8d3f29bf-3a63-47b5-9640-600d72bab00b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:45Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "openshift-service-ca.crt", "reconcileID": "8d3f29bf-3a63-47b5-9640-600d72bab00b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:45Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "eecc6534-247b-4b3c-aa02-b88edec43790", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:45Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "eecc6534-247b-4b3c-aa02-b88edec43790", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:46Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "2c123140-c70c-4d6a-921f-e75a7eee3e41", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:46Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "2c123140-c70c-4d6a-921f-e75a7eee3e41", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9"} 2026-07-10T22:56:46Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "2c123140-c70c-4d6a-921f-e75a7eee3e41", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:46Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "2c123140-c70c-4d6a-921f-e75a7eee3e41", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:46Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "2c123140-c70c-4d6a-921f-e75a7eee3e41", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9-kserve-route-authn"} 2026-07-10T22:56:46Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "2c123140-c70c-4d6a-921f-e75a7eee3e41", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9-kserve-route-authn"} 2026-07-10T22:56:46Z DEBUG Triggering LLMInferenceService Cleanup for Namespace {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "2c123140-c70c-4d6a-921f-e75a7eee3e41", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:46Z DEBUG Cleaning up LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "2c123140-c70c-4d6a-921f-e75a7eee3e41", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:46Z INFO Stop LLMInferenceService reconciliation {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-lifecycle-model-727180d9","namespace":"ai-tenant-e2e-mt-727180d9"}, "namespace": "ai-tenant-e2e-mt-727180d9", "name": "e2e-lifecycle-model-727180d9", "reconcileID": "21f6454a-df0e-4cae-a7a3-dcdc39faa424", "LLMInferenceService": "e2e-lifecycle-model-727180d9", "namespace": "ai-tenant-e2e-mt-727180d9"} 2026-07-10T22:56:46Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "265f07c8-ab65-4649-9482-2ebaee6f10ea", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:56:46Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "265f07c8-ab65-4649-9482-2ebaee6f10ea", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress", "name": "e2e-mt-727180d9-authn-ssl"} 2026-07-10T22:56:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "265f07c8-ab65-4649-9482-2ebaee6f10ea", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:56:51Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-mt-727180d9","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-mt-727180d9", "reconcileID": "721aeea9-1f7c-4da3-9bf8-ca231e7f59b0", "Gateway": "e2e-mt-727180d9", "namespace": "openshift-ingress"} 2026-07-10T22:56:52Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-6d12c532"}, "namespace": "ai-tenant-e2e-mt-6d12c532", "name": "openshift-service-ca.crt", "reconcileID": "2251a7b8-395a-4fc6-8d2a-5469d72c7eaa", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-6d12c532"} 2026-07-10T22:56:52Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-6d12c532"}, "namespace": "ai-tenant-e2e-mt-6d12c532", "name": "odh-trusted-ca-bundle", "reconcileID": "bca197ed-37bc-4714-ac62-a1f645b46120", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-6d12c532"} 2026-07-10T22:56:52Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-6d12c532"}, "namespace": "ai-tenant-e2e-mt-6d12c532", "name": "openshift-service-ca.crt", "reconcileID": "cd229b24-a7de-48d3-8959-2a6667d710b4", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-6d12c532", "create": "odh-kserve-custom-ca-bundle"} 2026-07-10T22:56:52Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-6d12c532"}, "namespace": "ai-tenant-e2e-mt-6d12c532", "name": "openshift-service-ca.crt", "reconcileID": "cd229b24-a7de-48d3-8959-2a6667d710b4", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-6d12c532"} 2026-07-10T22:56:52Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-6d12c532"}, "namespace": "ai-tenant-e2e-mt-6d12c532", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "361576ce-29fa-488e-a9c8-974a86f28499", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-6d12c532"} 2026-07-10T22:56:52Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-6d12c532"}, "namespace": "ai-tenant-e2e-mt-6d12c532", "name": "openshift-service-ca.crt", "reconcileID": "39872ac6-0366-4571-abc3-783eb23bbe3a", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-6d12c532"} 2026-07-10T22:56:52Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-6d12c532"}, "namespace": "ai-tenant-e2e-mt-6d12c532", "name": "odh-trusted-ca-bundle", "reconcileID": "c57132c9-f0d3-49e2-9199-82344e6bdbb9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-6d12c532", "update": "odh-kserve-custom-ca-bundle"} 2026-07-10T22:56:53Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-6d12c532"}, "namespace": "ai-tenant-e2e-mt-6d12c532", "name": "odh-trusted-ca-bundle", "reconcileID": "c57132c9-f0d3-49e2-9199-82344e6bdbb9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-6d12c532"} 2026-07-10T22:56:53Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-6d12c532"}, "namespace": "ai-tenant-e2e-mt-6d12c532", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a190ec6d-095b-48ef-920e-36c1baf6176b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-6d12c532"} 2026-07-10T22:56:53Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a7b1512a"}, "namespace": "ai-tenant-e2e-mt-a7b1512a", "name": "openshift-service-ca.crt", "reconcileID": "97d81f57-9578-4223-bdb2-d878cdf54658", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a7b1512a"} 2026-07-10T22:56:53Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a7b1512a"}, "namespace": "ai-tenant-e2e-mt-a7b1512a", "name": "odh-trusted-ca-bundle", "reconcileID": "8962f4e3-ec50-4645-ad6b-9278611099fd", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a7b1512a"} 2026-07-10T22:56:53Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a7b1512a"}, "namespace": "ai-tenant-e2e-mt-a7b1512a", "name": "openshift-service-ca.crt", "reconcileID": "5d2294cb-73ed-43e9-a0b1-0f2555479bc5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a7b1512a", "create": "odh-kserve-custom-ca-bundle"} 2026-07-10T22:56:53Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a7b1512a"}, "namespace": "ai-tenant-e2e-mt-a7b1512a", "name": "openshift-service-ca.crt", "reconcileID": "5d2294cb-73ed-43e9-a0b1-0f2555479bc5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a7b1512a"} 2026-07-10T22:56:53Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a7b1512a"}, "namespace": "ai-tenant-e2e-mt-a7b1512a", "name": "openshift-service-ca.crt", "reconcileID": "25b544a0-e1dd-4f2e-a391-b9ab6e0ad929", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a7b1512a", "create": "odh-kserve-custom-ca-bundle"} 2026-07-10T22:56:53Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a7b1512a"}, "namespace": "ai-tenant-e2e-mt-a7b1512a", "name": "openshift-service-ca.crt", "reconcileID": "25b544a0-e1dd-4f2e-a391-b9ab6e0ad929", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-10T22:56:53Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a7b1512a"}, "namespace": "ai-tenant-e2e-mt-a7b1512a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "991f6909-1063-49fa-8fd7-cb173eac087d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a7b1512a"} 2026-07-10T22:56:53Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a7b1512a"}, "namespace": "ai-tenant-e2e-mt-a7b1512a", "name": "openshift-service-ca.crt", "reconcileID": "4d6dd973-d81b-4ae7-af3c-4838488a7499", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a7b1512a"} 2026-07-10T22:56:53Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a7b1512a"}, "namespace": "ai-tenant-e2e-mt-a7b1512a", "name": "odh-trusted-ca-bundle", "reconcileID": "7da03476-c891-4c94-9df3-e8da706b3198", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a7b1512a", "update": "odh-kserve-custom-ca-bundle"} 2026-07-10T22:56:53Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a7b1512a"}, "namespace": "ai-tenant-e2e-mt-a7b1512a", "name": "odh-trusted-ca-bundle", "reconcileID": "7da03476-c891-4c94-9df3-e8da706b3198", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a7b1512a"} 2026-07-10T22:56:53Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a7b1512a"}, "namespace": "ai-tenant-e2e-mt-a7b1512a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cc5722bf-69d1-4595-8168-eaa78c463066", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a7b1512a"} 2026-07-10T22:57:00Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-6d12c532"}, "namespace": "ai-tenant-e2e-mt-6d12c532", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "47f11c9d-9aa7-40aa-842f-d3c16af3ac6f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-6d12c532"} 2026-07-10T22:57:00Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-6d12c532"}, "namespace": "ai-tenant-e2e-mt-6d12c532", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "47f11c9d-9aa7-40aa-842f-d3c16af3ac6f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-6d12c532", "create": "odh-kserve-custom-ca-bundle"} 2026-07-10T22:57:00Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-6d12c532"}, "namespace": "ai-tenant-e2e-mt-6d12c532", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "47f11c9d-9aa7-40aa-842f-d3c16af3ac6f", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-6d12c532 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-10T22:57:00Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-6d12c532"}, "namespace": "ai-tenant-e2e-mt-6d12c532", "name": "odh-trusted-ca-bundle", "reconcileID": "d66966da-5c8c-40e3-b0ac-fc5b1245cb11", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-6d12c532"} 2026-07-10T22:57:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-6d12c532"}, "namespace": "ai-tenant-e2e-mt-6d12c532", "name": "odh-trusted-ca-bundle", "reconcileID": "d66966da-5c8c-40e3-b0ac-fc5b1245cb11", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-6d12c532"} 2026-07-10T22:57:00Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-6d12c532"}, "namespace": "ai-tenant-e2e-mt-6d12c532", "name": "openshift-service-ca.crt", "reconcileID": "e3d25cfc-87d6-4f55-99ff-cf5cfaa3daa3", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-6d12c532"} 2026-07-10T22:57:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-6d12c532"}, "namespace": "ai-tenant-e2e-mt-6d12c532", "name": "openshift-service-ca.crt", "reconcileID": "e3d25cfc-87d6-4f55-99ff-cf5cfaa3daa3", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-6d12c532"} 2026-07-10T22:57:00Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-6d12c532"}, "namespace": "ai-tenant-e2e-mt-6d12c532", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f442e967-f052-4650-93fa-fee52a67ad70", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-6d12c532"} 2026-07-10T22:57:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-6d12c532"}, "namespace": "ai-tenant-e2e-mt-6d12c532", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f442e967-f052-4650-93fa-fee52a67ad70", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-6d12c532"} 2026-07-10T22:57:07Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a7b1512a"}, "namespace": "ai-tenant-e2e-mt-a7b1512a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2e46db34-e17f-4475-821d-90d8689346e8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a7b1512a"} 2026-07-10T22:57:07Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a7b1512a"}, "namespace": "ai-tenant-e2e-mt-a7b1512a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2e46db34-e17f-4475-821d-90d8689346e8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a7b1512a", "create": "odh-kserve-custom-ca-bundle"} 2026-07-10T22:57:07Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a7b1512a"}, "namespace": "ai-tenant-e2e-mt-a7b1512a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2e46db34-e17f-4475-821d-90d8689346e8", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-a7b1512a because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-10T22:57:07Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a7b1512a"}, "namespace": "ai-tenant-e2e-mt-a7b1512a", "name": "odh-trusted-ca-bundle", "reconcileID": "d67913ce-9cc2-4dab-9a29-cd014193f14a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a7b1512a"} 2026-07-10T22:57:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-a7b1512a"}, "namespace": "ai-tenant-e2e-mt-a7b1512a", "name": "odh-trusted-ca-bundle", "reconcileID": "d67913ce-9cc2-4dab-9a29-cd014193f14a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-a7b1512a"} 2026-07-10T22:57:07Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a7b1512a"}, "namespace": "ai-tenant-e2e-mt-a7b1512a", "name": "openshift-service-ca.crt", "reconcileID": "6d6660ff-1974-4264-96c8-e794094f5d17", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a7b1512a"} 2026-07-10T22:57:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-a7b1512a"}, "namespace": "ai-tenant-e2e-mt-a7b1512a", "name": "openshift-service-ca.crt", "reconcileID": "6d6660ff-1974-4264-96c8-e794094f5d17", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-a7b1512a"} 2026-07-10T22:57:07Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a7b1512a"}, "namespace": "ai-tenant-e2e-mt-a7b1512a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7fd1e3d7-14a6-4ed7-9078-b1c8c6bfce0a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a7b1512a"} 2026-07-10T22:57:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-a7b1512a"}, "namespace": "ai-tenant-e2e-mt-a7b1512a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7fd1e3d7-14a6-4ed7-9078-b1c8c6bfce0a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-a7b1512a"} 2026-07-10T22:57:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9f4af73a"}, "namespace": "ai-tenant-e2e-mt-9f4af73a", "name": "openshift-service-ca.crt", "reconcileID": "0fcb9267-7c6a-4995-ad05-7283fa1fc099", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9f4af73a"} 2026-07-10T22:57:07Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9f4af73a"}, "namespace": "ai-tenant-e2e-mt-9f4af73a", "name": "openshift-service-ca.crt", "reconcileID": "bf8a89b3-b6ef-44a3-87a4-fecf7796b10d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9f4af73a", "create": "odh-kserve-custom-ca-bundle"} 2026-07-10T22:57:07Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9f4af73a"}, "namespace": "ai-tenant-e2e-mt-9f4af73a", "name": "openshift-service-ca.crt", "reconcileID": "bf8a89b3-b6ef-44a3-87a4-fecf7796b10d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9f4af73a"} 2026-07-10T22:57:07Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-9f4af73a"}, "namespace": "ai-tenant-e2e-mt-9f4af73a", "name": "odh-trusted-ca-bundle", "reconcileID": "75adab8d-3c68-4e38-9a1a-36e8fea04ce8", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-9f4af73a", "create": "odh-kserve-custom-ca-bundle"} 2026-07-10T22:57:07Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-9f4af73a"}, "namespace": "ai-tenant-e2e-mt-9f4af73a", "name": "odh-trusted-ca-bundle", "reconcileID": "75adab8d-3c68-4e38-9a1a-36e8fea04ce8", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-10T22:57:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9f4af73a"}, "namespace": "ai-tenant-e2e-mt-9f4af73a", "name": "openshift-service-ca.crt", "reconcileID": "82c004a5-fe90-4c23-8dce-401465702143", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9f4af73a"} 2026-07-10T22:57:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9f4af73a"}, "namespace": "ai-tenant-e2e-mt-9f4af73a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "15bdf060-a941-4477-8a70-e0f0f9642887", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9f4af73a"} 2026-07-10T22:57:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-9f4af73a"}, "namespace": "ai-tenant-e2e-mt-9f4af73a", "name": "odh-trusted-ca-bundle", "reconcileID": "0ea4e718-d57a-480e-aff9-1e6403e2a8bc", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-9f4af73a"} 2026-07-10T22:57:07Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-9f4af73a"}, "namespace": "ai-tenant-e2e-mt-9f4af73a", "name": "odh-trusted-ca-bundle", "reconcileID": "ebe454b3-90d9-40c5-8f3b-33885214be8e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-9f4af73a", "update": "odh-kserve-custom-ca-bundle"} 2026-07-10T22:57:07Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-9f4af73a"}, "namespace": "ai-tenant-e2e-mt-9f4af73a", "name": "odh-trusted-ca-bundle", "reconcileID": "ebe454b3-90d9-40c5-8f3b-33885214be8e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-9f4af73a"} 2026-07-10T22:57:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9f4af73a"}, "namespace": "ai-tenant-e2e-mt-9f4af73a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "34b8e299-dd1d-4eb4-80ee-e8c783cb0203", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9f4af73a"} 2026-07-10T22:57:45Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9f4af73a"}, "namespace": "ai-tenant-e2e-mt-9f4af73a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "751e32b0-c609-490e-8a7b-c24924532f03", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9f4af73a"} 2026-07-10T22:57:45Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9f4af73a"}, "namespace": "ai-tenant-e2e-mt-9f4af73a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "751e32b0-c609-490e-8a7b-c24924532f03", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9f4af73a", "create": "odh-kserve-custom-ca-bundle"} 2026-07-10T22:57:45Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9f4af73a"}, "namespace": "ai-tenant-e2e-mt-9f4af73a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "751e32b0-c609-490e-8a7b-c24924532f03", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-mt-9f4af73a because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-10T22:57:45Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-9f4af73a"}, "namespace": "ai-tenant-e2e-mt-9f4af73a", "name": "odh-trusted-ca-bundle", "reconcileID": "1f3c88c3-0498-4119-9fd3-8e9173caaafa", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-9f4af73a"} 2026-07-10T22:57:45Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-mt-9f4af73a"}, "namespace": "ai-tenant-e2e-mt-9f4af73a", "name": "odh-trusted-ca-bundle", "reconcileID": "1f3c88c3-0498-4119-9fd3-8e9173caaafa", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-mt-9f4af73a"} 2026-07-10T22:57:45Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9f4af73a"}, "namespace": "ai-tenant-e2e-mt-9f4af73a", "name": "openshift-service-ca.crt", "reconcileID": "a9b72833-36b7-41d0-8e78-878f2651d56e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9f4af73a"} 2026-07-10T22:57:45Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-mt-9f4af73a"}, "namespace": "ai-tenant-e2e-mt-9f4af73a", "name": "openshift-service-ca.crt", "reconcileID": "a9b72833-36b7-41d0-8e78-878f2651d56e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-mt-9f4af73a"} 2026-07-10T22:57:45Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9f4af73a"}, "namespace": "ai-tenant-e2e-mt-9f4af73a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3e0dbf24-d7cb-4a5c-b9ad-7ab5343b7271", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9f4af73a"} 2026-07-10T22:57:45Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-mt-9f4af73a"}, "namespace": "ai-tenant-e2e-mt-9f4af73a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3e0dbf24-d7cb-4a5c-b9ad-7ab5343b7271", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-mt-9f4af73a"} 2026-07-10T22:57:46Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "b1a3ca34-84dc-4a76-bd88-e7425f548b7c", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:57:46Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "b1a3ca34-84dc-4a76-bd88-e7425f548b7c", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c-authn-ssl"} 2026-07-10T22:57:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "b1a3ca34-84dc-4a76-bd88-e7425f548b7c", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:57:46Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "65c51e04-0ca1-426e-aff1-6b5c2b6e8526", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:57:46Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "65c51e04-0ca1-426e-aff1-6b5c2b6e8526", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c-authn-ssl"} 2026-07-10T22:57:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "65c51e04-0ca1-426e-aff1-6b5c2b6e8526", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:57:46Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "63a68f1d-33da-4bad-a0b6-65d430f7ecf7", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:57:46Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "63a68f1d-33da-4bad-a0b6-65d430f7ecf7", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c-authn-ssl"} 2026-07-10T22:57:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "63a68f1d-33da-4bad-a0b6-65d430f7ecf7", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:57:46Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "e9213424-42a3-41ef-b90d-7ccf1a65c061", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:57:46Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "e9213424-42a3-41ef-b90d-7ccf1a65c061", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c-authn-ssl"} 2026-07-10T22:57:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "e9213424-42a3-41ef-b90d-7ccf1a65c061", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:57:46Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "e5b2fef8-8674-48f8-ba6d-eaee29eb8141", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:57:46Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "e5b2fef8-8674-48f8-ba6d-eaee29eb8141", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c-authn-ssl"} 2026-07-10T22:57:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "e5b2fef8-8674-48f8-ba6d-eaee29eb8141", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:57:46Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "80303ec0-afe2-468a-85a6-82641294fd07", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:57:46Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "80303ec0-afe2-468a-85a6-82641294fd07", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c-authn-ssl"} 2026-07-10T22:57:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "80303ec0-afe2-468a-85a6-82641294fd07", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:57:47Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "a7515deb-12fe-4ee9-9edb-bff67df502d9", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:57:47Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "a7515deb-12fe-4ee9-9edb-bff67df502d9", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c-authn-ssl"} 2026-07-10T22:57:47Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "a7515deb-12fe-4ee9-9edb-bff67df502d9", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:57:49Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "openshift-service-ca.crt", "reconcileID": "9aaf5485-d734-4953-8432-5a6f4b2c26ce", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:57:49Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "odh-trusted-ca-bundle", "reconcileID": "61ce02a9-e633-4f62-a3bb-4c103b364d1b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:57:49Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "openshift-service-ca.crt", "reconcileID": "6a89f283-197e-4cb5-b4ab-c28990c77c3e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-c8c45c", "create": "odh-kserve-custom-ca-bundle"} 2026-07-10T22:57:49Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "openshift-service-ca.crt", "reconcileID": "6a89f283-197e-4cb5-b4ab-c28990c77c3e", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:57:49Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "eca0d9d5-d88c-4df5-8a65-129badfef332", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:57:49Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "openshift-service-ca.crt", "reconcileID": "e5920e64-23dd-4dc0-a7d2-3771770b4ba0", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:57:49Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "odh-trusted-ca-bundle", "reconcileID": "774f990a-d496-4c14-9fbf-b391f1596f61", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-c8c45c", "update": "odh-kserve-custom-ca-bundle"} 2026-07-10T22:57:49Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "odh-trusted-ca-bundle", "reconcileID": "774f990a-d496-4c14-9fbf-b391f1596f61", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:57:49Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "abf92ca4-2bf2-40f8-9c0f-6d829423fef2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:57:50Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "9047570b-cc44-4cbc-b1a4-286c29692540", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:57:50Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "9047570b-cc44-4cbc-b1a4-286c29692540", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c-authn-ssl"} 2026-07-10T22:57:50Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "9047570b-cc44-4cbc-b1a4-286c29692540", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:57:50Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "6384ecb1-a1ef-4f45-9ec5-d61682f2981d", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-10T22:57:54Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "04a50476-afdc-4e66-ba26-befa1951c6e5", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:57:54Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "04a50476-afdc-4e66-ba26-befa1951c6e5", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05-authn-ssl"} 2026-07-10T22:57:54Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "04a50476-afdc-4e66-ba26-befa1951c6e5", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:57:54Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "ee176f96-08b6-4cdc-8a3c-e6a8a2b3881b", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:57:54Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "ee176f96-08b6-4cdc-8a3c-e6a8a2b3881b", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05-authn-ssl"} 2026-07-10T22:57:54Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "ee176f96-08b6-4cdc-8a3c-e6a8a2b3881b", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:57:54Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "2506abda-cb23-435a-949e-ff01d80767ea", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:57:54Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "2506abda-cb23-435a-949e-ff01d80767ea", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05-authn-ssl"} 2026-07-10T22:57:54Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "2506abda-cb23-435a-949e-ff01d80767ea", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:57:54Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "21b463af-57c7-45e5-a646-42faf3fbd2c5", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:57:54Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "21b463af-57c7-45e5-a646-42faf3fbd2c5", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05-authn-ssl"} 2026-07-10T22:57:54Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "21b463af-57c7-45e5-a646-42faf3fbd2c5", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:57:54Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "0345b9ce-02b3-43d1-b8b2-72e791620768", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:57:54Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "0345b9ce-02b3-43d1-b8b2-72e791620768", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05-authn-ssl"} 2026-07-10T22:57:54Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "0345b9ce-02b3-43d1-b8b2-72e791620768", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:57:54Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "0b2795d1-4110-48f3-bc17-0b621a39129d", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:57:54Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "0b2795d1-4110-48f3-bc17-0b621a39129d", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05-authn-ssl"} 2026-07-10T22:57:54Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "0b2795d1-4110-48f3-bc17-0b621a39129d", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:57:55Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "1f5ceae9-e2cc-44cb-8f4b-50cb92f23fe9", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:57:55Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "1f5ceae9-e2cc-44cb-8f4b-50cb92f23fe9", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05-authn-ssl"} 2026-07-10T22:57:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "1f5ceae9-e2cc-44cb-8f4b-50cb92f23fe9", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:57:57Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "openshift-service-ca.crt", "reconcileID": "163fcc17-70ec-4764-80c9-22d591653779", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:57:57Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "odh-trusted-ca-bundle", "reconcileID": "065f70d5-009c-4765-bbb9-90be7e55a6f4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:57:57Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "openshift-service-ca.crt", "reconcileID": "654ab409-bbf2-4a40-bca1-65f8f85f2af3", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-c49b05", "create": "odh-kserve-custom-ca-bundle"} 2026-07-10T22:57:57Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "openshift-service-ca.crt", "reconcileID": "654ab409-bbf2-4a40-bca1-65f8f85f2af3", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:57:57Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "openshift-service-ca.crt", "reconcileID": "999193c4-08a0-4762-9a42-9a10b82b86aa", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:57:57Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "87eb992f-cc29-4cc2-9f66-1de390dcdfd8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:57:57Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "odh-trusted-ca-bundle", "reconcileID": "ba71feb9-0548-4773-9226-aaf6ce19ddc0", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-c49b05", "update": "odh-kserve-custom-ca-bundle"} 2026-07-10T22:57:57Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "odh-trusted-ca-bundle", "reconcileID": "ba71feb9-0548-4773-9226-aaf6ce19ddc0", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:57:57Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "eb604aa0-2804-4b00-b1bb-663da7eed5c9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:57:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "6fb90e40-436f-4fba-a8da-882885b6ade4", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:57:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "6fb90e40-436f-4fba-a8da-882885b6ade4", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05-authn-ssl"} 2026-07-10T22:57:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "6fb90e40-436f-4fba-a8da-882885b6ade4", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:57:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "d168924b-ad7f-4263-bffe-f417b764d583", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-10T22:57:58Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "7773c3bc-a8ad-4bff-9ad6-301868082a85", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:57:58Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "7773c3bc-a8ad-4bff-9ad6-301868082a85", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c-authn-ssl"} 2026-07-10T22:57:58Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "7773c3bc-a8ad-4bff-9ad6-301868082a85", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:58:02Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "87857895-b1ca-4b17-bcac-ace69b469192", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:58:02Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "87857895-b1ca-4b17-bcac-ace69b469192", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:58:02Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "87857895-b1ca-4b17-bcac-ace69b469192", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:58:02Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "972e996d-1707-485f-9a8a-9f21b43fdc13", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:58:02Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "87857895-b1ca-4b17-bcac-ace69b469192", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c-kserve-route-authn"} 2026-07-10T22:58:02Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "87857895-b1ca-4b17-bcac-ace69b469192", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c", "authType": "userdefined"} 2026-07-10T22:58:02Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "87857895-b1ca-4b17-bcac-ace69b469192", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:58:02Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "972e996d-1707-485f-9a8a-9f21b43fdc13", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c-authn-ssl"} 2026-07-10T22:58:02Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "972e996d-1707-485f-9a8a-9f21b43fdc13", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:58:02Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "c875361d-a391-48c0-aff2-8162d7ec8f2c", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:58:02Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "c875361d-a391-48c0-aff2-8162d7ec8f2c", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:58:02Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "c875361d-a391-48c0-aff2-8162d7ec8f2c", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:58:02Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "c875361d-a391-48c0-aff2-8162d7ec8f2c", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c-kserve-route-authn"} 2026-07-10T22:58:02Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "c875361d-a391-48c0-aff2-8162d7ec8f2c", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c", "authType": "userdefined"} 2026-07-10T22:58:02Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "c875361d-a391-48c0-aff2-8162d7ec8f2c", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:58:04Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "779f00e0-15c2-4e45-b5c8-50ac3a5df606", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:58:04Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "779f00e0-15c2-4e45-b5c8-50ac3a5df606", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05-authn-ssl"} 2026-07-10T22:58:04Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "779f00e0-15c2-4e45-b5c8-50ac3a5df606", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:58:09Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "ff787e3e-dc23-4623-84f7-d9838eeea1fa", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:58:09Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "ff787e3e-dc23-4623-84f7-d9838eeea1fa", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c-authn-ssl"} 2026-07-10T22:58:09Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "ff787e3e-dc23-4623-84f7-d9838eeea1fa", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:58:10Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "3280462a-4a52-432b-bab5-deb27c969b6c", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:58:10Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "3280462a-4a52-432b-bab5-deb27c969b6c", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:58:10Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "3280462a-4a52-432b-bab5-deb27c969b6c", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:58:10Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "3280462a-4a52-432b-bab5-deb27c969b6c", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c-kserve-route-authn"} 2026-07-10T22:58:10Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "3280462a-4a52-432b-bab5-deb27c969b6c", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c", "authType": "userdefined"} 2026-07-10T22:58:10Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "3280462a-4a52-432b-bab5-deb27c969b6c", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:58:21Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "f379401f-e4c9-44b7-977d-9ef6a17317b2", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:58:21Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "f379401f-e4c9-44b7-977d-9ef6a17317b2", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c-authn-ssl"} 2026-07-10T22:58:21Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "f379401f-e4c9-44b7-977d-9ef6a17317b2", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:58:25Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "0ab5f695-4d56-4a1f-831f-d79db82b22f9", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:58:25Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "0ab5f695-4d56-4a1f-831f-d79db82b22f9", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:58:25Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "0ab5f695-4d56-4a1f-831f-d79db82b22f9", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:58:25Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "0ab5f695-4d56-4a1f-831f-d79db82b22f9", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c-kserve-route-authn"} 2026-07-10T22:58:25Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "0ab5f695-4d56-4a1f-831f-d79db82b22f9", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c", "authType": "userdefined"} 2026-07-10T22:58:25Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "0ab5f695-4d56-4a1f-831f-d79db82b22f9", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:58:28Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "67f5f7a7-a105-4705-93a4-97851711e97c", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:58:28Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "67f5f7a7-a105-4705-93a4-97851711e97c", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:58:28Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "67f5f7a7-a105-4705-93a4-97851711e97c", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:58:28Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "9317dfdc-9257-4865-939b-324b178167ac", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:58:28Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "67f5f7a7-a105-4705-93a4-97851711e97c", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05-kserve-route-authn"} 2026-07-10T22:58:28Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "67f5f7a7-a105-4705-93a4-97851711e97c", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05", "authType": "userdefined"} 2026-07-10T22:58:28Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "67f5f7a7-a105-4705-93a4-97851711e97c", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:58:28Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "9317dfdc-9257-4865-939b-324b178167ac", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05-authn-ssl"} 2026-07-10T22:58:28Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "9317dfdc-9257-4865-939b-324b178167ac", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:58:51Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "4b7886b9-f61a-442f-ab0c-add07ec949e0", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:58:51Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "4b7886b9-f61a-442f-ab0c-add07ec949e0", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:58:51Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "4b7886b9-f61a-442f-ab0c-add07ec949e0", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:58:51Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "4b7886b9-f61a-442f-ab0c-add07ec949e0", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05-kserve-route-authn"} 2026-07-10T22:58:51Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "4b7886b9-f61a-442f-ab0c-add07ec949e0", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05", "authType": "userdefined"} 2026-07-10T22:58:51Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "4b7886b9-f61a-442f-ab0c-add07ec949e0", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:59:03Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "014414f6-983a-4bc7-b9d2-63a5724f0347", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:59:03Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "014414f6-983a-4bc7-b9d2-63a5724f0347", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05-authn-ssl"} 2026-07-10T22:59:03Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "014414f6-983a-4bc7-b9d2-63a5724f0347", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:59:05Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "0d8b1a2a-21b0-466a-9f81-15e3a0f64904", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:59:05Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "0d8b1a2a-21b0-466a-9f81-15e3a0f64904", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:59:05Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "0d8b1a2a-21b0-466a-9f81-15e3a0f64904", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:59:05Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "0d8b1a2a-21b0-466a-9f81-15e3a0f64904", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05-kserve-route-authn"} 2026-07-10T22:59:05Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "0d8b1a2a-21b0-466a-9f81-15e3a0f64904", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05", "authType": "userdefined"} 2026-07-10T22:59:05Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "0d8b1a2a-21b0-466a-9f81-15e3a0f64904", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:59:10Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "3f8d56a8-bdda-4186-96e6-0e6b68e4648b", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:59:10Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "3f8d56a8-bdda-4186-96e6-0e6b68e4648b", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05-authn-ssl"} 2026-07-10T22:59:10Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "3f8d56a8-bdda-4186-96e6-0e6b68e4648b", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:59:28Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "07f99e2a-081d-4c6e-9a3c-ff7d57225cf4", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:59:28Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "07f99e2a-081d-4c6e-9a3c-ff7d57225cf4", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:59:28Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "07f99e2a-081d-4c6e-9a3c-ff7d57225cf4", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:59:28Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "07f99e2a-081d-4c6e-9a3c-ff7d57225cf4", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05-kserve-route-authn"} 2026-07-10T22:59:28Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "07f99e2a-081d-4c6e-9a3c-ff7d57225cf4", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05", "authType": "userdefined"} 2026-07-10T22:59:28Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "07f99e2a-081d-4c6e-9a3c-ff7d57225cf4", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:59:36Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "ec343654-55f4-4764-a9d2-f2385ac6389a", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:59:36Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "ec343654-55f4-4764-a9d2-f2385ac6389a", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c"} 2026-07-10T22:59:36Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "ec343654-55f4-4764-a9d2-f2385ac6389a", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:59:36Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "ec343654-55f4-4764-a9d2-f2385ac6389a", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:59:36Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "ec343654-55f4-4764-a9d2-f2385ac6389a", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c-kserve-route-authn"} 2026-07-10T22:59:36Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "ec343654-55f4-4764-a9d2-f2385ac6389a", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c-kserve-route-authn"} 2026-07-10T22:59:36Z DEBUG Triggering LLMInferenceService Cleanup for Namespace {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "ec343654-55f4-4764-a9d2-f2385ac6389a", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:59:36Z DEBUG Cleaning up LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "ec343654-55f4-4764-a9d2-f2385ac6389a", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:59:36Z INFO Stop LLMInferenceService reconciliation {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c8c45c","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "test-model-c8c45c", "reconcileID": "939a45d1-f751-4e1f-874f-4e19b00b3593", "LLMInferenceService": "test-model-c8c45c", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T22:59:36Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "e46a9074-39f6-405d-9cf3-5ede20b22044", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:59:36Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "e46a9074-39f6-405d-9cf3-5ede20b22044", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c-authn-ssl"} 2026-07-10T22:59:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "e46a9074-39f6-405d-9cf3-5ede20b22044", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:59:36Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "84754e63-a7e3-44a2-aac2-17330b89ddb5", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:59:36Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "84754e63-a7e3-44a2-aac2-17330b89ddb5", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c-authn-ssl"} 2026-07-10T22:59:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "84754e63-a7e3-44a2-aac2-17330b89ddb5", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:59:37Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "5bc7de1d-afde-4ea4-a159-bd1c267057c4", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:59:37Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "5bc7de1d-afde-4ea4-a159-bd1c267057c4", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05"} 2026-07-10T22:59:37Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "5bc7de1d-afde-4ea4-a159-bd1c267057c4", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:59:37Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "5bc7de1d-afde-4ea4-a159-bd1c267057c4", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:59:37Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "5bc7de1d-afde-4ea4-a159-bd1c267057c4", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05-kserve-route-authn"} 2026-07-10T22:59:37Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "5bc7de1d-afde-4ea4-a159-bd1c267057c4", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05-kserve-route-authn"} 2026-07-10T22:59:37Z DEBUG Triggering LLMInferenceService Cleanup for Namespace {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "5bc7de1d-afde-4ea4-a159-bd1c267057c4", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:59:37Z DEBUG Cleaning up LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "5bc7de1d-afde-4ea4-a159-bd1c267057c4", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:59:37Z INFO Stop LLMInferenceService reconciliation {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-c49b05","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "test-model-c49b05", "reconcileID": "2a3d286c-a04d-487a-bb75-b2778c56aa6e", "LLMInferenceService": "test-model-c49b05", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T22:59:37Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "5ce50bfa-c796-445f-b9b2-ed825de33431", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:59:37Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "5ce50bfa-c796-445f-b9b2-ed825de33431", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05-authn-ssl"} 2026-07-10T22:59:37Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "5ce50bfa-c796-445f-b9b2-ed825de33431", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:59:37Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "7a8551fa-3995-49d7-aa7c-a88a9ed40f52", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:59:37Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "7a8551fa-3995-49d7-aa7c-a88a9ed40f52", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05-authn-ssl"} 2026-07-10T22:59:37Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "7a8551fa-3995-49d7-aa7c-a88a9ed40f52", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T22:59:37Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "4dc138b5-7c2a-4643-9ab8-359c6b009001", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T22:59:37Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "4dc138b5-7c2a-4643-9ab8-359c6b009001", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c-authn-ssl"} 2026-07-10T22:59:37Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "4dc138b5-7c2a-4643-9ab8-359c6b009001", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T23:00:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4b37bcfe-357d-4bee-957f-ff4dc965cba2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T23:00:19Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4b37bcfe-357d-4bee-957f-ff4dc965cba2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-c8c45c", "create": "odh-kserve-custom-ca-bundle"} 2026-07-10T23:00:19Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4b37bcfe-357d-4bee-957f-ff4dc965cba2", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-inf-a-c8c45c because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-10T23:00:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "odh-trusted-ca-bundle", "reconcileID": "6db923f6-af26-4b94-b521-96ac8352f1b7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T23:00:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "odh-trusted-ca-bundle", "reconcileID": "6db923f6-af26-4b94-b521-96ac8352f1b7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T23:00:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "openshift-service-ca.crt", "reconcileID": "ebbb4437-2fae-4508-bfb8-1ad2f600b330", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T23:00:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "openshift-service-ca.crt", "reconcileID": "ebbb4437-2fae-4508-bfb8-1ad2f600b330", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T23:00:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c6f5fcdc-d26e-4cc3-9d93-709a1d4ef124", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T23:00:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-c8c45c"}, "namespace": "ai-tenant-e2e-inf-a-c8c45c", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c6f5fcdc-d26e-4cc3-9d93-709a1d4ef124", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-c8c45c"} 2026-07-10T23:00:20Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-c8c45c","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-c8c45c", "reconcileID": "d5436d16-3fbf-4f66-83b2-270ff1621680", "Gateway": "e2e-inf-a-c8c45c", "namespace": "openshift-ingress"} 2026-07-10T23:00:20Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "ff1da8af-f600-4645-8f1b-4ea1e5138d2d", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T23:00:20Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "ff1da8af-f600-4645-8f1b-4ea1e5138d2d", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05-authn-ssl"} 2026-07-10T23:00:20Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "ff1da8af-f600-4645-8f1b-4ea1e5138d2d", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T23:00:26Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ffa882e2-7b9f-449c-919e-fbae2db22432", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T23:00:26Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ffa882e2-7b9f-449c-919e-fbae2db22432", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-c49b05", "create": "odh-kserve-custom-ca-bundle"} 2026-07-10T23:00:26Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ffa882e2-7b9f-449c-919e-fbae2db22432", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-inf-b-c49b05 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-10T23:00:26Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "odh-trusted-ca-bundle", "reconcileID": "58183cb7-4436-438d-81de-0719a0f15a4a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T23:00:26Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "odh-trusted-ca-bundle", "reconcileID": "58183cb7-4436-438d-81de-0719a0f15a4a", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T23:00:26Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "openshift-service-ca.crt", "reconcileID": "aade6080-efe3-41cc-8e5f-9a6589833296", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T23:00:26Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "openshift-service-ca.crt", "reconcileID": "aade6080-efe3-41cc-8e5f-9a6589833296", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T23:00:26Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "10a1c276-1924-47f3-9594-f8d04d9bf776", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T23:00:26Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-c49b05"}, "namespace": "ai-tenant-e2e-inf-b-c49b05", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "10a1c276-1924-47f3-9594-f8d04d9bf776", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-c49b05"} 2026-07-10T23:00:26Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-10T23:00:26Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-10T23:00:27Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-c49b05","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-c49b05", "reconcileID": "971e3a8c-1ba9-4f7b-bdf2-896608535253", "Gateway": "e2e-inf-b-c49b05", "namespace": "openshift-ingress"} 2026-07-10T23:00:27Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-54ec41","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41", "reconcileID": "031d3319-1fa4-4419-9bea-701103d6b854", "Gateway": "e2e-api-a-54ec41", "namespace": "openshift-ingress"} 2026-07-10T23:00:27Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-54ec41","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41", "reconcileID": "031d3319-1fa4-4419-9bea-701103d6b854", "Gateway": "e2e-api-a-54ec41", "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41-authn-ssl"} 2026-07-10T23:00:27Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-54ec41","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41", "reconcileID": "031d3319-1fa4-4419-9bea-701103d6b854", "Gateway": "e2e-api-a-54ec41", "namespace": "openshift-ingress"} 2026-07-10T23:00:27Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-54ec41","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41", "reconcileID": "b6d98f11-e225-4d4d-948a-1dda27527189", "Gateway": "e2e-api-a-54ec41", "namespace": "openshift-ingress"} 2026-07-10T23:00:27Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-54ec41","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41", "reconcileID": "b6d98f11-e225-4d4d-948a-1dda27527189", "Gateway": "e2e-api-a-54ec41", "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41-authn-ssl"} 2026-07-10T23:00:27Z ERROR Reconciler error {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-54ec41","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41", "reconcileID": "b6d98f11-e225-4d4d-948a-1dda27527189", "error": "failed to create EnvoyFilter: could not CREATE EnvoyFilter openshift-ingress/e2e-api-a-54ec41-authn-ssl: envoyfilters.networking.istio.io \"e2e-api-a-54ec41-authn-ssl\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-10T23:00:27Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-54ec41","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41", "reconcileID": "a5b34695-850b-4702-bc69-530eec9bef66", "Gateway": "e2e-api-a-54ec41", "namespace": "openshift-ingress"} 2026-07-10T23:00:27Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-54ec41","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41", "reconcileID": "a5b34695-850b-4702-bc69-530eec9bef66", "Gateway": "e2e-api-a-54ec41", "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41-authn-ssl"} 2026-07-10T23:00:27Z DEBUG events Failed to reconcile EnvoyFilter: failed to create EnvoyFilter: could not CREATE EnvoyFilter openshift-ingress/e2e-api-a-54ec41-authn-ssl: envoyfilters.networking.istio.io "e2e-api-a-54ec41-authn-ssl" already exists {"type": "Warning", "object": {"kind":"Gateway","namespace":"openshift-ingress","name":"e2e-api-a-54ec41","uid":"469dbe8b-fa6b-4e1e-90ff-75a5d0c12a87","apiVersion":"gateway.networking.k8s.io/v1","resourceVersion":"52348"}, "reason": "ReconcileError"} 2026-07-10T23:00:27Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-54ec41","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41", "reconcileID": "a5b34695-850b-4702-bc69-530eec9bef66", "Gateway": "e2e-api-a-54ec41", "namespace": "openshift-ingress"} 2026-07-10T23:00:27Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-54ec41","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41", "reconcileID": "155072ca-c228-4c26-ab16-52c90f1412ca", "Gateway": "e2e-api-a-54ec41", "namespace": "openshift-ingress"} 2026-07-10T23:00:27Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-54ec41","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41", "reconcileID": "155072ca-c228-4c26-ab16-52c90f1412ca", "Gateway": "e2e-api-a-54ec41", "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41-authn-ssl"} 2026-07-10T23:00:27Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-54ec41","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41", "reconcileID": "155072ca-c228-4c26-ab16-52c90f1412ca", "Gateway": "e2e-api-a-54ec41", "namespace": "openshift-ingress"} 2026-07-10T23:00:28Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-54ec41","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41", "reconcileID": "118e06a2-777f-4086-876f-259ea44745d2", "Gateway": "e2e-api-a-54ec41", "namespace": "openshift-ingress"} 2026-07-10T23:00:28Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-54ec41","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41", "reconcileID": "118e06a2-777f-4086-876f-259ea44745d2", "Gateway": "e2e-api-a-54ec41", "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41-authn-ssl"} 2026-07-10T23:00:28Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-54ec41","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41", "reconcileID": "118e06a2-777f-4086-876f-259ea44745d2", "Gateway": "e2e-api-a-54ec41", "namespace": "openshift-ingress"} 2026-07-10T23:00:28Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-54ec41","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41", "reconcileID": "054ed06c-a728-4981-a2f1-c3595bf3ba43", "Gateway": "e2e-api-a-54ec41", "namespace": "openshift-ingress"} 2026-07-10T23:00:28Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-54ec41","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41", "reconcileID": "054ed06c-a728-4981-a2f1-c3595bf3ba43", "Gateway": "e2e-api-a-54ec41", "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41-authn-ssl"} 2026-07-10T23:00:28Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-54ec41","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41", "reconcileID": "054ed06c-a728-4981-a2f1-c3595bf3ba43", "Gateway": "e2e-api-a-54ec41", "namespace": "openshift-ingress"} 2026-07-10T23:00:29Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-54ec41","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41", "reconcileID": "82911546-8a74-4ec1-b9b9-2daf7298dc7f", "Gateway": "e2e-api-a-54ec41", "namespace": "openshift-ingress"} 2026-07-10T23:00:29Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-54ec41","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41", "reconcileID": "82911546-8a74-4ec1-b9b9-2daf7298dc7f", "Gateway": "e2e-api-a-54ec41", "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41-authn-ssl"} 2026-07-10T23:00:29Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-54ec41","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41", "reconcileID": "82911546-8a74-4ec1-b9b9-2daf7298dc7f", "Gateway": "e2e-api-a-54ec41", "namespace": "openshift-ingress"} 2026-07-10T23:00:31Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-54ec41"}, "namespace": "ai-tenant-e2e-api-a-54ec41", "name": "openshift-service-ca.crt", "reconcileID": "cb1c45d7-74ef-4c86-990f-dc405817ea19", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-54ec41", "create": "odh-kserve-custom-ca-bundle"} 2026-07-10T23:00:31Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-54ec41"}, "namespace": "ai-tenant-e2e-api-a-54ec41", "name": "openshift-service-ca.crt", "reconcileID": "cb1c45d7-74ef-4c86-990f-dc405817ea19", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-54ec41"} 2026-07-10T23:00:31Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-54ec41"}, "namespace": "ai-tenant-e2e-api-a-54ec41", "name": "odh-trusted-ca-bundle", "reconcileID": "fd76155e-1f39-44d0-9824-49de06c97d99", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-54ec41", "create": "odh-kserve-custom-ca-bundle"} 2026-07-10T23:00:31Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-54ec41"}, "namespace": "ai-tenant-e2e-api-a-54ec41", "name": "odh-trusted-ca-bundle", "reconcileID": "fd76155e-1f39-44d0-9824-49de06c97d99", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-10T23:00:31Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-54ec41"}, "namespace": "ai-tenant-e2e-api-a-54ec41", "name": "openshift-service-ca.crt", "reconcileID": "d7800c37-0ea8-41e8-b718-d319628b1c79", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-54ec41", "update": "odh-kserve-custom-ca-bundle"} 2026-07-10T23:00:31Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-54ec41"}, "namespace": "ai-tenant-e2e-api-a-54ec41", "name": "openshift-service-ca.crt", "reconcileID": "d7800c37-0ea8-41e8-b718-d319628b1c79", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-54ec41"} 2026-07-10T23:00:31Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-54ec41"}, "namespace": "ai-tenant-e2e-api-a-54ec41", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6bbdd6cb-8f3e-436a-bd09-5e68e86b4d12", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-54ec41"} 2026-07-10T23:00:31Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-54ec41"}, "namespace": "ai-tenant-e2e-api-a-54ec41", "name": "odh-trusted-ca-bundle", "reconcileID": "f3e3a9fd-48eb-498d-b63c-a4206700b415", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-54ec41"} 2026-07-10T23:00:32Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-10T23:00:32Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-07-10T23:03:46Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-54ec41"}, "namespace": "ai-tenant-e2e-api-a-54ec41", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cc1d11a1-4a6b-4ea6-8112-77d4d6d0b7f2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-54ec41"} 2026-07-10T23:03:46Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-54ec41"}, "namespace": "ai-tenant-e2e-api-a-54ec41", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cc1d11a1-4a6b-4ea6-8112-77d4d6d0b7f2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-54ec41", "create": "odh-kserve-custom-ca-bundle"} 2026-07-10T23:03:46Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-54ec41"}, "namespace": "ai-tenant-e2e-api-a-54ec41", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cc1d11a1-4a6b-4ea6-8112-77d4d6d0b7f2", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-api-a-54ec41 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-10T23:03:46Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-54ec41"}, "namespace": "ai-tenant-e2e-api-a-54ec41", "name": "odh-trusted-ca-bundle", "reconcileID": "815d06ed-7111-4c8f-9d7f-ee10b6fb0366", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-54ec41"} 2026-07-10T23:03:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-54ec41"}, "namespace": "ai-tenant-e2e-api-a-54ec41", "name": "odh-trusted-ca-bundle", "reconcileID": "815d06ed-7111-4c8f-9d7f-ee10b6fb0366", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-54ec41"} 2026-07-10T23:03:46Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-54ec41"}, "namespace": "ai-tenant-e2e-api-a-54ec41", "name": "openshift-service-ca.crt", "reconcileID": "9871f5e0-aaa9-4fa1-ab10-d1c0007fbc4c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-54ec41"} 2026-07-10T23:03:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-54ec41"}, "namespace": "ai-tenant-e2e-api-a-54ec41", "name": "openshift-service-ca.crt", "reconcileID": "9871f5e0-aaa9-4fa1-ab10-d1c0007fbc4c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-54ec41"} 2026-07-10T23:03:46Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-54ec41"}, "namespace": "ai-tenant-e2e-api-a-54ec41", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e30a0bd7-998b-4e27-b0a0-38520cb5c6df", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-54ec41"} 2026-07-10T23:03:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-54ec41"}, "namespace": "ai-tenant-e2e-api-a-54ec41", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e30a0bd7-998b-4e27-b0a0-38520cb5c6df", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-54ec41"} 2026-07-10T23:03:47Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-54ec41","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-54ec41", "reconcileID": "93973868-3337-43e4-817b-635929dc19ab", "Gateway": "e2e-api-a-54ec41", "namespace": "openshift-ingress"}