sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-09-07T17:21:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-b36e3a"}, "namespace": "ai-tenant-e2e-api-a-b36e3a", "name": "odh-trusted-ca-bundle", "reconcileID": "0e3d9d94-cba0-44d8-a86c-d561c14cd3b9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-b36e3a"} 2026-09-07T17:21:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-b36e3a"}, "namespace": "ai-tenant-e2e-api-a-b36e3a", "name": "odh-trusted-ca-bundle", "reconcileID": "0e3d9d94-cba0-44d8-a86c-d561c14cd3b9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-b36e3a"} 2026-09-07T17:21:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-b36e3a"}, "namespace": "ai-tenant-e2e-api-a-b36e3a", "name": "openshift-service-ca.crt", "reconcileID": "0e16bc60-d921-4d91-a91d-46f89fe74280", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-b36e3a"} 2026-09-07T17:21:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-b36e3a"}, "namespace": "ai-tenant-e2e-api-a-b36e3a", "name": "openshift-service-ca.crt", "reconcileID": "0e16bc60-d921-4d91-a91d-46f89fe74280", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-b36e3a"} 2026-09-07T17:21:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-b36e3a"}, "namespace": "ai-tenant-e2e-api-a-b36e3a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9a8eb74d-116c-41fa-adae-34d6d0922647", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-b36e3a"} 2026-09-07T17:21:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-b36e3a"}, "namespace": "ai-tenant-e2e-api-a-b36e3a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9a8eb74d-116c-41fa-adae-34d6d0922647", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-b36e3a"} 2026-09-07T17:21:17Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-3973bf","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "rate-test-model-3973bf", "reconcileID": "5683e176-7bfc-4b12-8e7b-c3b3f027a478", "LLMInferenceService": "rate-test-model-3973bf", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf"} 2026-09-07T17:21:17Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-3973bf","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "rate-test-model-3973bf", "reconcileID": "5683e176-7bfc-4b12-8e7b-c3b3f027a478", "LLMInferenceService": "rate-test-model-3973bf", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf"} 2026-09-07T17:21:17Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-3973bf","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "rate-test-model-3973bf", "reconcileID": "5683e176-7bfc-4b12-8e7b-c3b3f027a478", "LLMInferenceService": "rate-test-model-3973bf", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf"} 2026-09-07T17:21:17Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-3973bf","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "rate-test-model-3973bf", "reconcileID": "5683e176-7bfc-4b12-8e7b-c3b3f027a478", "LLMInferenceService": "rate-test-model-3973bf", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "rate-test-model-3973bf-kserve-route-authn"} 2026-09-07T17:21:17Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-3973bf","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "rate-test-model-3973bf", "reconcileID": "5683e176-7bfc-4b12-8e7b-c3b3f027a478", "LLMInferenceService": "rate-test-model-3973bf", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "authType": "userdefined"} 2026-09-07T17:21:17Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-3973bf","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "rate-test-model-3973bf", "reconcileID": "5683e176-7bfc-4b12-8e7b-c3b3f027a478", "LLMInferenceService": "rate-test-model-3973bf", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf"} 2026-09-07T17:21:17Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-b36e3a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-b36e3a", "reconcileID": "fb88d580-a3b3-46b2-a226-e4cb87e9d17e", "Gateway": "e2e-api-a-b36e3a", "namespace": "openshift-ingress"} 2026-09-07T17:21:34Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-611042","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-611042", "reconcileID": "a664119f-2f1d-4de8-900e-20d15595b74c", "Gateway": "e2e-api-b-611042", "namespace": "openshift-ingress"} 2026-09-07T17:21:34Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-611042","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-611042", "reconcileID": "a664119f-2f1d-4de8-900e-20d15595b74c", "Gateway": "e2e-api-b-611042", "namespace": "openshift-ingress", "name": "e2e-api-b-611042-authn-ssl"} 2026-09-07T17:21:34Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-611042","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-611042", "reconcileID": "a664119f-2f1d-4de8-900e-20d15595b74c", "Gateway": "e2e-api-b-611042", "namespace": "openshift-ingress"} 2026-09-07T17:21:39Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-611042"}, "namespace": "ai-tenant-e2e-api-b-611042", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "01e6b3dd-f22d-44f4-9ee4-d0c816d7dd0b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-b-611042"} 2026-09-07T17:21:39Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-611042"}, "namespace": "ai-tenant-e2e-api-b-611042", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "01e6b3dd-f22d-44f4-9ee4-d0c816d7dd0b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-b-611042", "create": "odh-kserve-custom-ca-bundle"} 2026-09-07T17:21:40Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-611042"}, "namespace": "ai-tenant-e2e-api-b-611042", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "01e6b3dd-f22d-44f4-9ee4-d0c816d7dd0b", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-api-b-611042 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-09-07T17:21:40Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-b-611042"}, "namespace": "ai-tenant-e2e-api-b-611042", "name": "odh-trusted-ca-bundle", "reconcileID": "2d7e825d-0fb7-465f-9aa7-d7e7123fcd43", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-b-611042"} 2026-09-07T17:21:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-b-611042"}, "namespace": "ai-tenant-e2e-api-b-611042", "name": "odh-trusted-ca-bundle", "reconcileID": "2d7e825d-0fb7-465f-9aa7-d7e7123fcd43", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-b-611042"} 2026-09-07T17:21:40Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-b-611042"}, "namespace": "ai-tenant-e2e-api-b-611042", "name": "openshift-service-ca.crt", "reconcileID": "8f3f4f78-ad61-4c5a-8823-ace2aeeb48e0", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-b-611042"} 2026-09-07T17:21:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-b-611042"}, "namespace": "ai-tenant-e2e-api-b-611042", "name": "openshift-service-ca.crt", "reconcileID": "8f3f4f78-ad61-4c5a-8823-ace2aeeb48e0", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-b-611042"} 2026-09-07T17:21:40Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-611042"}, "namespace": "ai-tenant-e2e-api-b-611042", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b17e3414-3c5d-41d8-b602-3c4cdba78fd2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-b-611042"} 2026-09-07T17:21:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-611042"}, "namespace": "ai-tenant-e2e-api-b-611042", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b17e3414-3c5d-41d8-b602-3c4cdba78fd2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-b-611042"} 2026-09-07T17:21:41Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-611042","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-611042", "reconcileID": "79a1205a-1b46-4db4-9854-e5cf0167e15d", "Gateway": "e2e-api-b-611042", "namespace": "openshift-ingress"} 2026-09-07T17:21:52Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-w2-202015","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-w2-202015", "reconcileID": "fdc2be41-169c-4b52-8db1-51eb1e1c2448", "Gateway": "e2e-shared-a-w2-202015", "namespace": "openshift-ingress"} 2026-09-07T17:21:52Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-w2-202015","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-w2-202015", "reconcileID": "fdc2be41-169c-4b52-8db1-51eb1e1c2448", "Gateway": "e2e-shared-a-w2-202015", "namespace": "openshift-ingress", "name": "e2e-shared-a-w2-202015-authn-ssl"} 2026-09-07T17:21:52Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-w2-202015","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-w2-202015", "reconcileID": "fdc2be41-169c-4b52-8db1-51eb1e1c2448", "Gateway": "e2e-shared-a-w2-202015", "namespace": "openshift-ingress"} 2026-09-07T17:21:52Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-1795dd","namespace":"ai-tenant-e2e-shared-a-w3-1795dd"}, "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "name": "rate-test-model-1795dd", "reconcileID": "fd9a37d2-0c01-42ab-bfaf-8ed041a5e96f", "LLMInferenceService": "rate-test-model-1795dd", "namespace": "ai-tenant-e2e-shared-a-w3-1795dd"} 2026-09-07T17:21:52Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-1795dd","namespace":"ai-tenant-e2e-shared-a-w3-1795dd"}, "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "name": "rate-test-model-1795dd", "reconcileID": "fd9a37d2-0c01-42ab-bfaf-8ed041a5e96f", "LLMInferenceService": "rate-test-model-1795dd", "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "name": "rate-test-model-1795dd"} 2026-09-07T17:21:52Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-1795dd","namespace":"ai-tenant-e2e-shared-a-w3-1795dd"}, "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "name": "rate-test-model-1795dd", "reconcileID": "fd9a37d2-0c01-42ab-bfaf-8ed041a5e96f", "LLMInferenceService": "rate-test-model-1795dd", "namespace": "ai-tenant-e2e-shared-a-w3-1795dd"} 2026-09-07T17:21:52Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-1795dd","namespace":"ai-tenant-e2e-shared-a-w3-1795dd"}, "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "name": "rate-test-model-1795dd", "reconcileID": "fd9a37d2-0c01-42ab-bfaf-8ed041a5e96f", "LLMInferenceService": "rate-test-model-1795dd", "namespace": "ai-tenant-e2e-shared-a-w3-1795dd"} 2026-09-07T17:21:52Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-1795dd","namespace":"ai-tenant-e2e-shared-a-w3-1795dd"}, "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "name": "rate-test-model-1795dd", "reconcileID": "fd9a37d2-0c01-42ab-bfaf-8ed041a5e96f", "LLMInferenceService": "rate-test-model-1795dd", "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "name": "rate-test-model-1795dd-kserve-route-authn"} 2026-09-07T17:21:52Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-1795dd","namespace":"ai-tenant-e2e-shared-a-w3-1795dd"}, "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "name": "rate-test-model-1795dd", "reconcileID": "fd9a37d2-0c01-42ab-bfaf-8ed041a5e96f", "LLMInferenceService": "rate-test-model-1795dd", "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "name": "rate-test-model-1795dd-kserve-route-authn"} 2026-09-07T17:21:52Z DEBUG Triggering LLMInferenceService Cleanup for Namespace {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-1795dd","namespace":"ai-tenant-e2e-shared-a-w3-1795dd"}, "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "name": "rate-test-model-1795dd", "reconcileID": "fd9a37d2-0c01-42ab-bfaf-8ed041a5e96f", "LLMInferenceService": "rate-test-model-1795dd", "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "namespace": "ai-tenant-e2e-shared-a-w3-1795dd"} 2026-09-07T17:21:52Z DEBUG Cleaning up LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-1795dd","namespace":"ai-tenant-e2e-shared-a-w3-1795dd"}, "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "name": "rate-test-model-1795dd", "reconcileID": "fd9a37d2-0c01-42ab-bfaf-8ed041a5e96f", "LLMInferenceService": "rate-test-model-1795dd", "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "namespace": "ai-tenant-e2e-shared-a-w3-1795dd"} 2026-09-07T17:21:54Z INFO Stop LLMInferenceService reconciliation {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-1795dd","namespace":"ai-tenant-e2e-shared-a-w3-1795dd"}, "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "name": "rate-test-model-1795dd", "reconcileID": "cc5aac85-11cd-4094-907b-b9607dd3543d", "LLMInferenceService": "rate-test-model-1795dd", "namespace": "ai-tenant-e2e-shared-a-w3-1795dd"} 2026-09-07T17:21:54Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-w3-1795dd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-w3-1795dd", "reconcileID": "c69df043-f802-4b30-b50d-d70e30ae6698", "Gateway": "e2e-shared-a-w3-1795dd", "namespace": "openshift-ingress"} 2026-09-07T17:21:54Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-w3-1795dd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-w3-1795dd", "reconcileID": "c69df043-f802-4b30-b50d-d70e30ae6698", "Gateway": "e2e-shared-a-w3-1795dd", "namespace": "openshift-ingress", "name": "e2e-shared-a-w3-1795dd-authn-ssl"} 2026-09-07T17:21:54Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-w3-1795dd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-w3-1795dd", "reconcileID": "c69df043-f802-4b30-b50d-d70e30ae6698", "Gateway": "e2e-shared-a-w3-1795dd", "namespace": "openshift-ingress"} 2026-09-07T17:21:54Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-w3-1795dd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-w3-1795dd", "reconcileID": "63a48da8-6795-4371-a322-5e6c86767c05", "Gateway": "e2e-shared-a-w3-1795dd", "namespace": "openshift-ingress"} 2026-09-07T17:21:54Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-w3-1795dd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-w3-1795dd", "reconcileID": "63a48da8-6795-4371-a322-5e6c86767c05", "Gateway": "e2e-shared-a-w3-1795dd", "namespace": "openshift-ingress", "name": "e2e-shared-a-w3-1795dd-authn-ssl"} 2026-09-07T17:21:54Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-w3-1795dd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-w3-1795dd", "reconcileID": "63a48da8-6795-4371-a322-5e6c86767c05", "Gateway": "e2e-shared-a-w3-1795dd", "namespace": "openshift-ingress"} 2026-09-07T17:21:54Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-3973bf","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "rate-test-model-3973bf", "reconcileID": "4aabb93b-b330-4333-8107-ad0f1e6a7aa4", "LLMInferenceService": "rate-test-model-3973bf", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf"} 2026-09-07T17:21:54Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-3973bf","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "rate-test-model-3973bf", "reconcileID": "4aabb93b-b330-4333-8107-ad0f1e6a7aa4", "LLMInferenceService": "rate-test-model-3973bf", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "rate-test-model-3973bf"} 2026-09-07T17:21:54Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-3973bf","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "rate-test-model-3973bf", "reconcileID": "4aabb93b-b330-4333-8107-ad0f1e6a7aa4", "LLMInferenceService": "rate-test-model-3973bf", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf"} 2026-09-07T17:21:54Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-3973bf","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "rate-test-model-3973bf", "reconcileID": "4aabb93b-b330-4333-8107-ad0f1e6a7aa4", "LLMInferenceService": "rate-test-model-3973bf", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf"} 2026-09-07T17:21:54Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-3973bf","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "rate-test-model-3973bf", "reconcileID": "4aabb93b-b330-4333-8107-ad0f1e6a7aa4", "LLMInferenceService": "rate-test-model-3973bf", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "rate-test-model-3973bf-kserve-route-authn"} 2026-09-07T17:21:54Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-3973bf","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "rate-test-model-3973bf", "reconcileID": "4aabb93b-b330-4333-8107-ad0f1e6a7aa4", "LLMInferenceService": "rate-test-model-3973bf", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "rate-test-model-3973bf-kserve-route-authn"} 2026-09-07T17:21:54Z DEBUG Triggering LLMInferenceService Cleanup for Namespace {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-3973bf","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "rate-test-model-3973bf", "reconcileID": "4aabb93b-b330-4333-8107-ad0f1e6a7aa4", "LLMInferenceService": "rate-test-model-3973bf", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf"} 2026-09-07T17:21:54Z DEBUG Cleaning up LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-3973bf","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "rate-test-model-3973bf", "reconcileID": "4aabb93b-b330-4333-8107-ad0f1e6a7aa4", "LLMInferenceService": "rate-test-model-3973bf", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf"} 2026-09-07T17:21:54Z INFO Stop LLMInferenceService reconciliation {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"rate-test-model-3973bf","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "rate-test-model-3973bf", "reconcileID": "83036484-d1ed-4cab-85cf-09e50de35592", "LLMInferenceService": "rate-test-model-3973bf", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf"} 2026-09-07T17:21:54Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-w3-3973bf","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-w3-3973bf", "reconcileID": "377804c2-98d4-4601-8c58-0b3d1f8ae464", "Gateway": "e2e-shared-b-w3-3973bf", "namespace": "openshift-ingress"} 2026-09-07T17:21:54Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-w3-3973bf","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-w3-3973bf", "reconcileID": "377804c2-98d4-4601-8c58-0b3d1f8ae464", "Gateway": "e2e-shared-b-w3-3973bf", "namespace": "openshift-ingress", "name": "e2e-shared-b-w3-3973bf-authn-ssl"} 2026-09-07T17:21:54Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-w3-3973bf","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-w3-3973bf", "reconcileID": "377804c2-98d4-4601-8c58-0b3d1f8ae464", "Gateway": "e2e-shared-b-w3-3973bf", "namespace": "openshift-ingress"} 2026-09-07T17:21:55Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-w3-3973bf","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-w3-3973bf", "reconcileID": "7d366cf5-25e1-4453-9433-b873a8732386", "Gateway": "e2e-shared-b-w3-3973bf", "namespace": "openshift-ingress"} 2026-09-07T17:21:55Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-w3-3973bf","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-w3-3973bf", "reconcileID": "7d366cf5-25e1-4453-9433-b873a8732386", "Gateway": "e2e-shared-b-w3-3973bf", "namespace": "openshift-ingress", "name": "e2e-shared-b-w3-3973bf-authn-ssl"} 2026-09-07T17:21:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-w3-3973bf","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-w3-3973bf", "reconcileID": "7d366cf5-25e1-4453-9433-b873a8732386", "Gateway": "e2e-shared-b-w3-3973bf", "namespace": "openshift-ingress"} 2026-09-07T17:21:55Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "989e765f-50d5-49d8-8d53-587e9097bd0b", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:21:55Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "989e765f-50d5-49d8-8d53-587e9097bd0b", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce-authn-ssl"} 2026-09-07T17:21:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "989e765f-50d5-49d8-8d53-587e9097bd0b", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:21:55Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "c2ceda37-e6f2-4c34-9e8b-ad04a95413c5", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:21:55Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "c2ceda37-e6f2-4c34-9e8b-ad04a95413c5", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce-authn-ssl"} 2026-09-07T17:21:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "c2ceda37-e6f2-4c34-9e8b-ad04a95413c5", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:21:55Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "89be2e58-90ae-49cf-993c-a1c100bc8874", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:21:55Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "89be2e58-90ae-49cf-993c-a1c100bc8874", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce-authn-ssl"} 2026-09-07T17:21:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "89be2e58-90ae-49cf-993c-a1c100bc8874", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:21:55Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "6bf6701a-5aed-4aa4-b2e9-105d0b30052e", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:21:55Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "6bf6701a-5aed-4aa4-b2e9-105d0b30052e", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce-authn-ssl"} 2026-09-07T17:21:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "6bf6701a-5aed-4aa4-b2e9-105d0b30052e", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:21:55Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "f0c0305b-f1ba-4281-a3bb-5e4a11a92c70", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:21:55Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "f0c0305b-f1ba-4281-a3bb-5e4a11a92c70", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce-authn-ssl"} 2026-09-07T17:21:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "f0c0305b-f1ba-4281-a3bb-5e4a11a92c70", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:21:55Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "95350263-08bd-452c-9de3-01a2529683dd", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:21:55Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "95350263-08bd-452c-9de3-01a2529683dd", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce-authn-ssl"} 2026-09-07T17:21:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "95350263-08bd-452c-9de3-01a2529683dd", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:21:56Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "1b7e41f6-39b6-457a-88de-b096f565d8cf", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:21:56Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "1b7e41f6-39b6-457a-88de-b096f565d8cf", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce-authn-ssl"} 2026-09-07T17:21:56Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "1b7e41f6-39b6-457a-88de-b096f565d8cf", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:21:56Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "e8378378-5e53-400c-8d51-471d6afa7bd4", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:21:56Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "e8378378-5e53-400c-8d51-471d6afa7bd4", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce-authn-ssl"} 2026-09-07T17:21:56Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "e8378378-5e53-400c-8d51-471d6afa7bd4", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:21:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "openshift-service-ca.crt", "reconcileID": "0c4b9dce-6bd7-402b-b934-a06e7c1e13f9", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:21:58Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "odh-trusted-ca-bundle", "reconcileID": "95144a83-fac2-4013-a217-29ce58f2f7a6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "create": "odh-kserve-custom-ca-bundle"} 2026-09-07T17:21:58Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "odh-trusted-ca-bundle", "reconcileID": "95144a83-fac2-4013-a217-29ce58f2f7a6", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:21:58Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "openshift-service-ca.crt", "reconcileID": "532164db-b44c-4bed-ac34-2144a623db70", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "create": "odh-kserve-custom-ca-bundle"} 2026-09-07T17:21:58Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "openshift-service-ca.crt", "reconcileID": "532164db-b44c-4bed-ac34-2144a623db70", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-09-07T17:21:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3cbc79f5-43df-472e-b7ef-2b1624ab6001", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:21:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "openshift-service-ca.crt", "reconcileID": "c2e2ed57-d1db-4cc4-b1e6-fc59e11504e7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:21:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "openshift-service-ca.crt", "reconcileID": "c636cb1d-bc07-4cbe-8f54-988765b2304b", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:21:58Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "odh-trusted-ca-bundle", "reconcileID": "f3b495f0-98a6-4edc-8e22-85ece4f81668", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "update": "odh-kserve-custom-ca-bundle"} 2026-09-07T17:21:58Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "odh-trusted-ca-bundle", "reconcileID": "f3b495f0-98a6-4edc-8e22-85ece4f81668", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:21:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bab91176-16c1-4a26-8f8e-c019d271d3d4", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:21:59Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-w2-202015"}, "namespace": "ai-tenant-e2e-shared-a-w2-202015", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cf2aaf7e-9c7b-4072-af4c-4de57988286c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-w2-202015"} 2026-09-07T17:21:59Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-w2-202015"}, "namespace": "ai-tenant-e2e-shared-a-w2-202015", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cf2aaf7e-9c7b-4072-af4c-4de57988286c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-w2-202015", "create": "odh-kserve-custom-ca-bundle"} 2026-09-07T17:21:59Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-w2-202015"}, "namespace": "ai-tenant-e2e-shared-a-w2-202015", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "cf2aaf7e-9c7b-4072-af4c-4de57988286c", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-w2-202015 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-09-07T17:21:59Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-a-w2-202015"}, "namespace": "ai-tenant-e2e-shared-a-w2-202015", "name": "odh-trusted-ca-bundle", "reconcileID": "b2dc2006-d015-4db1-85ca-551e435d3669", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-w2-202015"} 2026-09-07T17:21:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-a-w2-202015"}, "namespace": "ai-tenant-e2e-shared-a-w2-202015", "name": "odh-trusted-ca-bundle", "reconcileID": "b2dc2006-d015-4db1-85ca-551e435d3669", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-w2-202015"} 2026-09-07T17:21:59Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-w2-202015"}, "namespace": "ai-tenant-e2e-shared-a-w2-202015", "name": "openshift-service-ca.crt", "reconcileID": "337f8dbf-16d0-426e-9de0-c3939bc8989d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-a-w2-202015"} 2026-09-07T17:21:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-w2-202015"}, "namespace": "ai-tenant-e2e-shared-a-w2-202015", "name": "openshift-service-ca.crt", "reconcileID": "337f8dbf-16d0-426e-9de0-c3939bc8989d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-a-w2-202015"} 2026-09-07T17:21:59Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-w2-202015"}, "namespace": "ai-tenant-e2e-shared-a-w2-202015", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "300fd77f-5ecd-45d1-8a04-a31e3805552a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-w2-202015"} 2026-09-07T17:21:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-w2-202015"}, "namespace": "ai-tenant-e2e-shared-a-w2-202015", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "300fd77f-5ecd-45d1-8a04-a31e3805552a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-w2-202015"} 2026-09-07T17:21:59Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-w2-202015","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-w2-202015", "reconcileID": "0d1f5f80-3072-4640-89e6-6f9d434b0fc3", "Gateway": "e2e-shared-a-w2-202015", "namespace": "openshift-ingress"} 2026-09-07T17:21:59Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-w2-202015","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-w2-202015", "reconcileID": "0d1f5f80-3072-4640-89e6-6f9d434b0fc3", "Gateway": "e2e-shared-a-w2-202015", "namespace": "openshift-ingress", "name": "e2e-shared-a-w2-202015-authn-ssl"} 2026-09-07T17:21:59Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-w2-202015","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-w2-202015", "reconcileID": "0d1f5f80-3072-4640-89e6-6f9d434b0fc3", "Gateway": "e2e-shared-a-w2-202015", "namespace": "openshift-ingress"} 2026-09-07T17:21:59Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-w2-202015","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-w2-202015", "reconcileID": "91c5ede6-29bd-4162-a048-4519adbd7a76", "Gateway": "e2e-shared-a-w2-202015", "namespace": "openshift-ingress"} 2026-09-07T17:22:00Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "fde4c0cd-7122-47c4-b7d0-abdadf8eed2f", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:22:00Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "fde4c0cd-7122-47c4-b7d0-abdadf8eed2f", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce-authn-ssl"} 2026-09-07T17:22:00Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "fde4c0cd-7122-47c4-b7d0-abdadf8eed2f", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:22:05Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "73068d97-9cbf-4a42-972b-81ee1c682713", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:22:05Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "73068d97-9cbf-4a42-972b-81ee1c682713", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce-authn-ssl"} 2026-09-07T17:22:05Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "73068d97-9cbf-4a42-972b-81ee1c682713", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:22:10Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-w2-43ab35","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-w2-43ab35", "reconcileID": "3a589cbc-c6d5-4f39-9b4f-9a7c1bde5222", "Gateway": "e2e-shared-b-w2-43ab35", "namespace": "openshift-ingress"} 2026-09-07T17:22:10Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-w2-43ab35","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-w2-43ab35", "reconcileID": "3a589cbc-c6d5-4f39-9b4f-9a7c1bde5222", "Gateway": "e2e-shared-b-w2-43ab35", "namespace": "openshift-ingress", "name": "e2e-shared-b-w2-43ab35-authn-ssl"} 2026-09-07T17:22:10Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-w2-43ab35","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-w2-43ab35", "reconcileID": "3a589cbc-c6d5-4f39-9b4f-9a7c1bde5222", "Gateway": "e2e-shared-b-w2-43ab35", "namespace": "openshift-ingress"} 2026-09-07T17:22:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-w2-43ab35"}, "namespace": "ai-tenant-e2e-shared-b-w2-43ab35", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6665d9dc-5310-4098-84fa-1202b9335604", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-w2-43ab35"} 2026-09-07T17:22:16Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-w2-43ab35"}, "namespace": "ai-tenant-e2e-shared-b-w2-43ab35", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6665d9dc-5310-4098-84fa-1202b9335604", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-w2-43ab35", "create": "odh-kserve-custom-ca-bundle"} 2026-09-07T17:22:16Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-w2-43ab35"}, "namespace": "ai-tenant-e2e-shared-b-w2-43ab35", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6665d9dc-5310-4098-84fa-1202b9335604", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w2-43ab35 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-09-07T17:22:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-b-w2-43ab35"}, "namespace": "ai-tenant-e2e-shared-b-w2-43ab35", "name": "odh-trusted-ca-bundle", "reconcileID": "993a9d84-f818-45d7-b988-b59b337f5eb2", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-w2-43ab35"} 2026-09-07T17:22:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-b-w2-43ab35"}, "namespace": "ai-tenant-e2e-shared-b-w2-43ab35", "name": "odh-trusted-ca-bundle", "reconcileID": "993a9d84-f818-45d7-b988-b59b337f5eb2", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-w2-43ab35"} 2026-09-07T17:22:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-b-w2-43ab35"}, "namespace": "ai-tenant-e2e-shared-b-w2-43ab35", "name": "openshift-service-ca.crt", "reconcileID": "0f32b522-3081-4c30-9caa-69a2f1350cf7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-b-w2-43ab35"} 2026-09-07T17:22:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-b-w2-43ab35"}, "namespace": "ai-tenant-e2e-shared-b-w2-43ab35", "name": "openshift-service-ca.crt", "reconcileID": "0f32b522-3081-4c30-9caa-69a2f1350cf7", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-b-w2-43ab35"} 2026-09-07T17:22:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-w2-43ab35"}, "namespace": "ai-tenant-e2e-shared-b-w2-43ab35", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4894f5a4-791f-4fef-a609-7fffee7c0e7d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-w2-43ab35"} 2026-09-07T17:22:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-w2-43ab35"}, "namespace": "ai-tenant-e2e-shared-b-w2-43ab35", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4894f5a4-791f-4fef-a609-7fffee7c0e7d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-w2-43ab35"} 2026-09-07T17:22:17Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-w2-43ab35","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-w2-43ab35", "reconcileID": "6e38b8ee-bf60-4f86-b024-2058d7f0fcbf", "Gateway": "e2e-shared-b-w2-43ab35", "namespace": "openshift-ingress"} 2026-09-07T17:22:17Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-w2-43ab35","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-w2-43ab35", "reconcileID": "6e38b8ee-bf60-4f86-b024-2058d7f0fcbf", "Gateway": "e2e-shared-b-w2-43ab35", "namespace": "openshift-ingress", "name": "e2e-shared-b-w2-43ab35-authn-ssl"} 2026-09-07T17:22:17Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-w2-43ab35","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-w2-43ab35", "reconcileID": "6e38b8ee-bf60-4f86-b024-2058d7f0fcbf", "Gateway": "e2e-shared-b-w2-43ab35", "namespace": "openshift-ingress"} 2026-09-07T17:22:18Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-w2-43ab35","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-w2-43ab35", "reconcileID": "a08fcbbc-9b1d-4bfd-b531-c6f499998b2d", "Gateway": "e2e-shared-b-w2-43ab35", "namespace": "openshift-ingress"} 2026-09-07T17:22:24Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "a2d22762-9eb6-4dc3-a6f1-4aacec2f359f", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:22:24Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "a2d22762-9eb6-4dc3-a6f1-4aacec2f359f", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d-authn-ssl"} 2026-09-07T17:22:24Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "a2d22762-9eb6-4dc3-a6f1-4aacec2f359f", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:22:24Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "e0efe87f-c862-44e7-b9f8-00ee19823891", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:22:24Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "e0efe87f-c862-44e7-b9f8-00ee19823891", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d-authn-ssl"} 2026-09-07T17:22:24Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "e0efe87f-c862-44e7-b9f8-00ee19823891", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:22:24Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "f833f41c-d117-4a48-8669-79c864133dc7", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:22:24Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "f833f41c-d117-4a48-8669-79c864133dc7", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d-authn-ssl"} 2026-09-07T17:22:24Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "f833f41c-d117-4a48-8669-79c864133dc7", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:22:24Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "1beee79a-825f-43e6-b013-b8e1733d4126", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:22:24Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "1beee79a-825f-43e6-b013-b8e1733d4126", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d-authn-ssl"} 2026-09-07T17:22:24Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "1beee79a-825f-43e6-b013-b8e1733d4126", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:22:24Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "dcded11d-56c1-4bdf-9616-c34e47aaa026", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:22:24Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "dcded11d-56c1-4bdf-9616-c34e47aaa026", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d-authn-ssl"} 2026-09-07T17:22:24Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "dcded11d-56c1-4bdf-9616-c34e47aaa026", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:22:24Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "6b172063-f3ef-4652-92b1-ac418c10af56", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:22:24Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "6b172063-f3ef-4652-92b1-ac418c10af56", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d-authn-ssl"} 2026-09-07T17:22:24Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "6b172063-f3ef-4652-92b1-ac418c10af56", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:22:24Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "c26fd5dc-427f-4886-8c3f-ca4477504248", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:22:24Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "c26fd5dc-427f-4886-8c3f-ca4477504248", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d-authn-ssl"} 2026-09-07T17:22:24Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "c26fd5dc-427f-4886-8c3f-ca4477504248", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:22:26Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "97f5cd2e-6e2d-4b3d-b55d-ab8f56ee1210", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:22:26Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "97f5cd2e-6e2d-4b3d-b55d-ab8f56ee1210", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d-authn-ssl"} 2026-09-07T17:22:26Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "97f5cd2e-6e2d-4b3d-b55d-ab8f56ee1210", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:22:27Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ipp-b-fac82d"}, "namespace": "ai-tenant-e2e-ipp-b-fac82d", "name": "openshift-service-ca.crt", "reconcileID": "676716c9-06a3-4a11-9fd9-91d84f07c89d", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ipp-b-fac82d"} 2026-09-07T17:22:27Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ipp-b-fac82d"}, "namespace": "ai-tenant-e2e-ipp-b-fac82d", "name": "odh-trusted-ca-bundle", "reconcileID": "84fa0cf0-6866-41d1-bfca-1fc2eb08a7be", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ipp-b-fac82d"} 2026-09-07T17:22:27Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ipp-b-fac82d"}, "namespace": "ai-tenant-e2e-ipp-b-fac82d", "name": "openshift-service-ca.crt", "reconcileID": "394ce4ff-16e4-494f-a73e-59943a079868", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ipp-b-fac82d", "create": "odh-kserve-custom-ca-bundle"} 2026-09-07T17:22:27Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ipp-b-fac82d"}, "namespace": "ai-tenant-e2e-ipp-b-fac82d", "name": "openshift-service-ca.crt", "reconcileID": "394ce4ff-16e4-494f-a73e-59943a079868", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ipp-b-fac82d"} 2026-09-07T17:22:27Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ipp-b-fac82d"}, "namespace": "ai-tenant-e2e-ipp-b-fac82d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "74d0bfbe-e71a-44c8-91eb-dfd4f80bc152", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ipp-b-fac82d"} 2026-09-07T17:22:27Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ipp-b-fac82d"}, "namespace": "ai-tenant-e2e-ipp-b-fac82d", "name": "openshift-service-ca.crt", "reconcileID": "e6a9cec1-eb0a-4d96-91fe-f822cdc6ca23", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ipp-b-fac82d"} 2026-09-07T17:22:27Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ipp-b-fac82d"}, "namespace": "ai-tenant-e2e-ipp-b-fac82d", "name": "odh-trusted-ca-bundle", "reconcileID": "25ce4a5b-25a3-46bb-b62a-1ad352cb9df3", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ipp-b-fac82d", "update": "odh-kserve-custom-ca-bundle"} 2026-09-07T17:22:28Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ipp-b-fac82d"}, "namespace": "ai-tenant-e2e-ipp-b-fac82d", "name": "odh-trusted-ca-bundle", "reconcileID": "25ce4a5b-25a3-46bb-b62a-1ad352cb9df3", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ipp-b-fac82d"} 2026-09-07T17:22:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ipp-b-fac82d"}, "namespace": "ai-tenant-e2e-ipp-b-fac82d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bc567a85-c87f-488f-9830-6c2572c037ce", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ipp-b-fac82d"} 2026-09-07T17:22:29Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "45b19129-8545-437d-8ded-63213c15c668", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:22:29Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "45b19129-8545-437d-8ded-63213c15c668", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d-authn-ssl"} 2026-09-07T17:22:29Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "45b19129-8545-437d-8ded-63213c15c668", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:22:36Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "9faac055-741e-4662-a176-a289aa103a57", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:22:36Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "9faac055-741e-4662-a176-a289aa103a57", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d-authn-ssl"} 2026-09-07T17:22:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "9faac055-741e-4662-a176-a289aa103a57", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:22:39Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-09-07T17:22:39Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-09-07T17:22:40Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-09-07T17:22:40Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-09-07T17:22:58Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "25636e15-f52e-46dc-afe3-8707a867adec", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:22:58Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "25636e15-f52e-46dc-afe3-8707a867adec", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:22:58Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "25636e15-f52e-46dc-afe3-8707a867adec", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:22:58Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "f95b3ad5-9e51-4149-a819-5903afda7c92", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:22:58Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "25636e15-f52e-46dc-afe3-8707a867adec", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce-kserve-route-authn"} 2026-09-07T17:22:58Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "25636e15-f52e-46dc-afe3-8707a867adec", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "authType": "userdefined"} 2026-09-07T17:22:58Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "25636e15-f52e-46dc-afe3-8707a867adec", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:22:58Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "f95b3ad5-9e51-4149-a819-5903afda7c92", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce-authn-ssl"} 2026-09-07T17:22:58Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "f95b3ad5-9e51-4149-a819-5903afda7c92", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:22:58Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "264d2d7a-19cc-444e-9094-76570d4ef198", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:22:58Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "264d2d7a-19cc-444e-9094-76570d4ef198", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:22:58Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "264d2d7a-19cc-444e-9094-76570d4ef198", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:22:58Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "264d2d7a-19cc-444e-9094-76570d4ef198", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce-kserve-route-authn"} 2026-09-07T17:22:58Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "264d2d7a-19cc-444e-9094-76570d4ef198", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "authType": "userdefined"} 2026-09-07T17:22:58Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "264d2d7a-19cc-444e-9094-76570d4ef198", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:23:10Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "348d8058-e7d6-4763-a573-5f741e05f949", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:23:10Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "348d8058-e7d6-4763-a573-5f741e05f949", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce-authn-ssl"} 2026-09-07T17:23:10Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "348d8058-e7d6-4763-a573-5f741e05f949", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:23:12Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "3ae8db79-cf39-4f61-aba3-c646b88487eb", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:23:12Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "3ae8db79-cf39-4f61-aba3-c646b88487eb", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:23:12Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "3ae8db79-cf39-4f61-aba3-c646b88487eb", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:23:12Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "3ae8db79-cf39-4f61-aba3-c646b88487eb", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce-kserve-route-authn"} 2026-09-07T17:23:12Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "3ae8db79-cf39-4f61-aba3-c646b88487eb", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "authType": "userdefined"} 2026-09-07T17:23:12Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "3ae8db79-cf39-4f61-aba3-c646b88487eb", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:23:26Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "b80846ed-b45f-4717-8cef-8571192e9eee", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:23:26Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "b80846ed-b45f-4717-8cef-8571192e9eee", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:23:26Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "b80846ed-b45f-4717-8cef-8571192e9eee", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:23:26Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "b80846ed-b45f-4717-8cef-8571192e9eee", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce-kserve-route-authn"} 2026-09-07T17:23:26Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "b80846ed-b45f-4717-8cef-8571192e9eee", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "authType": "userdefined"} 2026-09-07T17:23:26Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "b80846ed-b45f-4717-8cef-8571192e9eee", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:23:39Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "0f5c9178-8cf0-4c4f-a1fa-4c4cd4671eb9", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress"} 2026-09-07T17:23:39Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "0f5c9178-8cf0-4c4f-a1fa-4c4cd4671eb9", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805-authn-ssl"} 2026-09-07T17:23:39Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "0f5c9178-8cf0-4c4f-a1fa-4c4cd4671eb9", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress"} 2026-09-07T17:23:39Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "9a84f196-87fa-4bf5-9c0d-ddffe7e4c34c", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress"} 2026-09-07T17:23:39Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "9a84f196-87fa-4bf5-9c0d-ddffe7e4c34c", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805-authn-ssl"} 2026-09-07T17:23:39Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "9a84f196-87fa-4bf5-9c0d-ddffe7e4c34c", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress"} 2026-09-07T17:23:39Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "7643b285-dd48-405f-a8de-3830a96cd62c", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress"} 2026-09-07T17:23:39Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "7643b285-dd48-405f-a8de-3830a96cd62c", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805-authn-ssl"} 2026-09-07T17:23:39Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "7643b285-dd48-405f-a8de-3830a96cd62c", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress"} 2026-09-07T17:23:39Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "7ff1ac9d-48a3-4ad9-8511-b3dba183e5ce", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress"} 2026-09-07T17:23:39Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "7ff1ac9d-48a3-4ad9-8511-b3dba183e5ce", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805-authn-ssl"} 2026-09-07T17:23:39Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "7ff1ac9d-48a3-4ad9-8511-b3dba183e5ce", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress"} 2026-09-07T17:23:39Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "f87e0c1b-35f2-4b53-a2be-8ed20443761a", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress"} 2026-09-07T17:23:39Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "f87e0c1b-35f2-4b53-a2be-8ed20443761a", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805-authn-ssl"} 2026-09-07T17:23:39Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "f87e0c1b-35f2-4b53-a2be-8ed20443761a", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress"} 2026-09-07T17:23:39Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "0cf6b9ac-e5e3-4524-8a76-4fdb181c0c40", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress"} 2026-09-07T17:23:39Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "0cf6b9ac-e5e3-4524-8a76-4fdb181c0c40", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805-authn-ssl"} 2026-09-07T17:23:39Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "0cf6b9ac-e5e3-4524-8a76-4fdb181c0c40", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress"} 2026-09-07T17:23:40Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "5b91049d-d32d-4d62-a141-63901807d2ef", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress"} 2026-09-07T17:23:40Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "5b91049d-d32d-4d62-a141-63901807d2ef", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805-authn-ssl"} 2026-09-07T17:23:40Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "5b91049d-d32d-4d62-a141-63901807d2ef", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress"} 2026-09-07T17:23:42Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ipp-cleanup-dad805"}, "namespace": "ai-tenant-e2e-ipp-cleanup-dad805", "name": "openshift-service-ca.crt", "reconcileID": "ae50848f-0d0f-469a-97c4-394491266652", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ipp-cleanup-dad805"} 2026-09-07T17:23:42Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ipp-cleanup-dad805"}, "namespace": "ai-tenant-e2e-ipp-cleanup-dad805", "name": "odh-trusted-ca-bundle", "reconcileID": "cb9859da-dac9-44ab-9046-847842a9af23", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ipp-cleanup-dad805"} 2026-09-07T17:23:42Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ipp-cleanup-dad805"}, "namespace": "ai-tenant-e2e-ipp-cleanup-dad805", "name": "openshift-service-ca.crt", "reconcileID": "1b12ae59-ffad-47f1-bc99-ec0bf7f97f11", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ipp-cleanup-dad805", "create": "odh-kserve-custom-ca-bundle"} 2026-09-07T17:23:42Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ipp-cleanup-dad805"}, "namespace": "ai-tenant-e2e-ipp-cleanup-dad805", "name": "openshift-service-ca.crt", "reconcileID": "1b12ae59-ffad-47f1-bc99-ec0bf7f97f11", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ipp-cleanup-dad805"} 2026-09-07T17:23:42Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ipp-cleanup-dad805"}, "namespace": "ai-tenant-e2e-ipp-cleanup-dad805", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ec205657-ebe8-46a6-9b0a-5acf83c0cd20", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ipp-cleanup-dad805"} 2026-09-07T17:23:42Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ipp-cleanup-dad805"}, "namespace": "ai-tenant-e2e-ipp-cleanup-dad805", "name": "openshift-service-ca.crt", "reconcileID": "69db2666-da01-40b9-a4e6-9c0f0d465ea0", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ipp-cleanup-dad805"} 2026-09-07T17:23:42Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ipp-cleanup-dad805"}, "namespace": "ai-tenant-e2e-ipp-cleanup-dad805", "name": "odh-trusted-ca-bundle", "reconcileID": "bda75d37-a10b-4c02-b570-d947b1136875", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ipp-cleanup-dad805", "update": "odh-kserve-custom-ca-bundle"} 2026-09-07T17:23:42Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ipp-cleanup-dad805"}, "namespace": "ai-tenant-e2e-ipp-cleanup-dad805", "name": "odh-trusted-ca-bundle", "reconcileID": "bda75d37-a10b-4c02-b570-d947b1136875", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ipp-cleanup-dad805"} 2026-09-07T17:23:42Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ipp-cleanup-dad805"}, "namespace": "ai-tenant-e2e-ipp-cleanup-dad805", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a4d8d4a1-23eb-4071-b197-4149519943c3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ipp-cleanup-dad805"} 2026-09-07T17:23:44Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "f41162ea-c42b-49c4-9c12-3f9a082e73a6", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress"} 2026-09-07T17:23:44Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "f41162ea-c42b-49c4-9c12-3f9a082e73a6", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805-authn-ssl"} 2026-09-07T17:23:44Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "f41162ea-c42b-49c4-9c12-3f9a082e73a6", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress"} 2026-09-07T17:23:52Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "e01a7a3a-0d4a-45ab-8eca-f31e795e4973", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress"} 2026-09-07T17:23:52Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "e01a7a3a-0d4a-45ab-8eca-f31e795e4973", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805-authn-ssl"} 2026-09-07T17:23:52Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "e01a7a3a-0d4a-45ab-8eca-f31e795e4973", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress"} 2026-09-07T17:24:14Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "831c8f50-9a34-4ac8-9ad8-92d74b2dfa87", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress"} 2026-09-07T17:24:14Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "831c8f50-9a34-4ac8-9ad8-92d74b2dfa87", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805-authn-ssl"} 2026-09-07T17:24:14Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "831c8f50-9a34-4ac8-9ad8-92d74b2dfa87", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress"} 2026-09-07T17:24:19Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "9e482a34-2bdb-4082-8be3-0e22a2a4a2f7", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress"} 2026-09-07T17:24:19Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "9e482a34-2bdb-4082-8be3-0e22a2a4a2f7", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805-authn-ssl"} 2026-09-07T17:24:19Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "9e482a34-2bdb-4082-8be3-0e22a2a4a2f7", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress"} 2026-09-07T17:24:20Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ipp-cleanup-dad805"}, "namespace": "ai-tenant-e2e-ipp-cleanup-dad805", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bdbcf166-64b6-4791-8478-763732ac7403", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ipp-cleanup-dad805"} 2026-09-07T17:24:20Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ipp-cleanup-dad805"}, "namespace": "ai-tenant-e2e-ipp-cleanup-dad805", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bdbcf166-64b6-4791-8478-763732ac7403", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ipp-cleanup-dad805", "create": "odh-kserve-custom-ca-bundle"} 2026-09-07T17:24:20Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ipp-cleanup-dad805"}, "namespace": "ai-tenant-e2e-ipp-cleanup-dad805", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "bdbcf166-64b6-4791-8478-763732ac7403", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ipp-cleanup-dad805 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-09-07T17:24:20Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ipp-cleanup-dad805"}, "namespace": "ai-tenant-e2e-ipp-cleanup-dad805", "name": "odh-trusted-ca-bundle", "reconcileID": "34a64021-a35f-424b-bb41-1700108c1f40", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ipp-cleanup-dad805"} 2026-09-07T17:24:20Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ipp-cleanup-dad805"}, "namespace": "ai-tenant-e2e-ipp-cleanup-dad805", "name": "odh-trusted-ca-bundle", "reconcileID": "34a64021-a35f-424b-bb41-1700108c1f40", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ipp-cleanup-dad805"} 2026-09-07T17:24:20Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ipp-cleanup-dad805"}, "namespace": "ai-tenant-e2e-ipp-cleanup-dad805", "name": "openshift-service-ca.crt", "reconcileID": "4cd677c3-19d8-4a4a-a6ce-5db8b3c083ae", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ipp-cleanup-dad805"} 2026-09-07T17:24:20Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ipp-cleanup-dad805"}, "namespace": "ai-tenant-e2e-ipp-cleanup-dad805", "name": "openshift-service-ca.crt", "reconcileID": "4cd677c3-19d8-4a4a-a6ce-5db8b3c083ae", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ipp-cleanup-dad805"} 2026-09-07T17:24:20Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ipp-cleanup-dad805"}, "namespace": "ai-tenant-e2e-ipp-cleanup-dad805", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "512b9914-91af-4017-af83-1e0c3d072005", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ipp-cleanup-dad805"} 2026-09-07T17:24:20Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ipp-cleanup-dad805"}, "namespace": "ai-tenant-e2e-ipp-cleanup-dad805", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "512b9914-91af-4017-af83-1e0c3d072005", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ipp-cleanup-dad805"} 2026-09-07T17:24:21Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-cleanup-dad805","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-cleanup-dad805", "reconcileID": "908d2a11-792f-4734-b4c6-64f8a23c31d1", "Gateway": "e2e-ipp-cleanup-dad805", "namespace": "openshift-ingress"} 2026-09-07T17:24:40Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "aaf22be6-bb2c-40dd-bf28-6fd579072667", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:24:40Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "aaf22be6-bb2c-40dd-bf28-6fd579072667", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce-authn-ssl"} 2026-09-07T17:24:40Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "aaf22be6-bb2c-40dd-bf28-6fd579072667", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:24:40Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "138e7309-c829-4c1b-aa54-f8252313d0c6", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:24:40Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "138e7309-c829-4c1b-aa54-f8252313d0c6", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce-authn-ssl"} 2026-09-07T17:24:40Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "138e7309-c829-4c1b-aa54-f8252313d0c6", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:24:46Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "7b728846-f26d-49eb-adef-1e8b7199bb58", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:24:46Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "7b728846-f26d-49eb-adef-1e8b7199bb58", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:24:46Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "7b728846-f26d-49eb-adef-1e8b7199bb58", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:24:46Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "7b728846-f26d-49eb-adef-1e8b7199bb58", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce-kserve-route-authn"} 2026-09-07T17:24:46Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "7b728846-f26d-49eb-adef-1e8b7199bb58", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "authType": "userdefined"} 2026-09-07T17:24:46Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "7b728846-f26d-49eb-adef-1e8b7199bb58", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:24:46Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "5bdcf18c-e191-4870-884d-8d7b2e434dc2", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:24:46Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "5bdcf18c-e191-4870-884d-8d7b2e434dc2", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce-authn-ssl"} 2026-09-07T17:24:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "5bdcf18c-e191-4870-884d-8d7b2e434dc2", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:25:22Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "887265e0-aa07-4140-9eab-76a3ec5dcbf8", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:25:22Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "887265e0-aa07-4140-9eab-76a3ec5dcbf8", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce"} 2026-09-07T17:25:22Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "887265e0-aa07-4140-9eab-76a3ec5dcbf8", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:25:22Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "887265e0-aa07-4140-9eab-76a3ec5dcbf8", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:25:22Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "887265e0-aa07-4140-9eab-76a3ec5dcbf8", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce-kserve-route-authn"} 2026-09-07T17:25:22Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "887265e0-aa07-4140-9eab-76a3ec5dcbf8", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce-kserve-route-authn"} 2026-09-07T17:25:22Z DEBUG Triggering LLMInferenceService Cleanup for Namespace {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "887265e0-aa07-4140-9eab-76a3ec5dcbf8", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:25:22Z DEBUG Cleaning up LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "887265e0-aa07-4140-9eab-76a3ec5dcbf8", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:25:22Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9ff41c1f-6ca1-4688-9145-684e97f30db0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:25:22Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9ff41c1f-6ca1-4688-9145-684e97f30db0", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "create": "odh-kserve-custom-ca-bundle"} 2026-09-07T17:25:22Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9ff41c1f-6ca1-4688-9145-684e97f30db0", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ipp-a-9765ce because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-09-07T17:25:22Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "odh-trusted-ca-bundle", "reconcileID": "c5354b30-7e2e-4ad9-9187-25ebba845012", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:25:22Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "odh-trusted-ca-bundle", "reconcileID": "c5354b30-7e2e-4ad9-9187-25ebba845012", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:25:22Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "openshift-service-ca.crt", "reconcileID": "49305899-719a-4157-b5a8-3c1b2034f552", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:25:22Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "openshift-service-ca.crt", "reconcileID": "49305899-719a-4157-b5a8-3c1b2034f552", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:25:22Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0f4cedc9-2d44-4379-a40f-0fd8d2958cd5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:25:22Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0f4cedc9-2d44-4379-a40f-0fd8d2958cd5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:25:23Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "e7d29ada-3420-4aab-bfe0-7fd482035838", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:25:23Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "e7d29ada-3420-4aab-bfe0-7fd482035838", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce-authn-ssl"} 2026-09-07T17:25:23Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "e7d29ada-3420-4aab-bfe0-7fd482035838", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:25:25Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "9bf38118-f654-47c9-9820-1e09c47f798e", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:25:25Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "9bf38118-f654-47c9-9820-1e09c47f798e", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce"} 2026-09-07T17:25:25Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "9bf38118-f654-47c9-9820-1e09c47f798e", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:25:25Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "9bf38118-f654-47c9-9820-1e09c47f798e", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:25:25Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "9bf38118-f654-47c9-9820-1e09c47f798e", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce-kserve-route-authn"} 2026-09-07T17:25:25Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "9bf38118-f654-47c9-9820-1e09c47f798e", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce-kserve-route-authn"} 2026-09-07T17:25:25Z DEBUG Triggering LLMInferenceService Cleanup for Namespace {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "9bf38118-f654-47c9-9820-1e09c47f798e", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:25:25Z DEBUG Cleaning up LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "9bf38118-f654-47c9-9820-1e09c47f798e", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:25:26Z INFO Stop LLMInferenceService reconciliation {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"ipp-route-9765ce","namespace":"ai-tenant-e2e-ipp-a-9765ce"}, "namespace": "ai-tenant-e2e-ipp-a-9765ce", "name": "ipp-route-9765ce", "reconcileID": "d685be51-ac84-428f-ab2d-e6182936e9cb", "LLMInferenceService": "ipp-route-9765ce", "namespace": "ai-tenant-e2e-ipp-a-9765ce"} 2026-09-07T17:25:26Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "b2b9b2ab-a607-4449-825b-ace1c8ee9ced", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:25:26Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "b2b9b2ab-a607-4449-825b-ace1c8ee9ced", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce-authn-ssl"} 2026-09-07T17:25:26Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "b2b9b2ab-a607-4449-825b-ace1c8ee9ced", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:25:29Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-a-9765ce","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-a-9765ce", "reconcileID": "73ce3b10-664a-4bb7-a71d-6b2be694cb3b", "Gateway": "e2e-ipp-a-9765ce", "namespace": "openshift-ingress"} 2026-09-07T17:25:40Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "ec5f9838-1c38-40fe-a5e6-3567707620ea", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:25:40Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "ec5f9838-1c38-40fe-a5e6-3567707620ea", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d-authn-ssl"} 2026-09-07T17:25:40Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "ec5f9838-1c38-40fe-a5e6-3567707620ea", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:25:46Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "b72f02c8-e9e0-49da-8f7d-0ffbccbe4ed3", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:25:46Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "b72f02c8-e9e0-49da-8f7d-0ffbccbe4ed3", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d-authn-ssl"} 2026-09-07T17:25:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "b72f02c8-e9e0-49da-8f7d-0ffbccbe4ed3", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:25:46Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ipp-b-fac82d"}, "namespace": "ai-tenant-e2e-ipp-b-fac82d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0c57dfd8-dc24-4b5b-ae83-d4fd36d6479a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ipp-b-fac82d"} 2026-09-07T17:25:46Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ipp-b-fac82d"}, "namespace": "ai-tenant-e2e-ipp-b-fac82d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0c57dfd8-dc24-4b5b-ae83-d4fd36d6479a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ipp-b-fac82d", "create": "odh-kserve-custom-ca-bundle"} 2026-09-07T17:25:46Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ipp-b-fac82d"}, "namespace": "ai-tenant-e2e-ipp-b-fac82d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0c57dfd8-dc24-4b5b-ae83-d4fd36d6479a", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ipp-b-fac82d because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-09-07T17:25:46Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ipp-b-fac82d"}, "namespace": "ai-tenant-e2e-ipp-b-fac82d", "name": "odh-trusted-ca-bundle", "reconcileID": "41362681-0f93-41b5-ae6d-dd7c0a1b3227", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ipp-b-fac82d"} 2026-09-07T17:25:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-ipp-b-fac82d"}, "namespace": "ai-tenant-e2e-ipp-b-fac82d", "name": "odh-trusted-ca-bundle", "reconcileID": "41362681-0f93-41b5-ae6d-dd7c0a1b3227", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-ipp-b-fac82d"} 2026-09-07T17:25:46Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ipp-b-fac82d"}, "namespace": "ai-tenant-e2e-ipp-b-fac82d", "name": "openshift-service-ca.crt", "reconcileID": "578d13eb-12b9-4c3e-838b-d4914317e51c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ipp-b-fac82d"} 2026-09-07T17:25:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-ipp-b-fac82d"}, "namespace": "ai-tenant-e2e-ipp-b-fac82d", "name": "openshift-service-ca.crt", "reconcileID": "578d13eb-12b9-4c3e-838b-d4914317e51c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-ipp-b-fac82d"} 2026-09-07T17:25:46Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ipp-b-fac82d"}, "namespace": "ai-tenant-e2e-ipp-b-fac82d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7e55c46a-523e-4604-ac47-e2d5516cbdc1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ipp-b-fac82d"} 2026-09-07T17:25:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-ipp-b-fac82d"}, "namespace": "ai-tenant-e2e-ipp-b-fac82d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7e55c46a-523e-4604-ac47-e2d5516cbdc1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-ipp-b-fac82d"} 2026-09-07T17:25:47Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ipp-b-fac82d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ipp-b-fac82d", "reconcileID": "66ec3611-bbbb-407b-99f2-186408731164", "Gateway": "e2e-ipp-b-fac82d", "namespace": "openshift-ingress"} 2026-09-07T17:25:58Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-w3-1795dd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-w3-1795dd", "reconcileID": "d2170881-2d26-4585-8e15-6adce9e11e50", "Gateway": "e2e-shared-a-w3-1795dd", "namespace": "openshift-ingress"} 2026-09-07T17:25:58Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-w3-1795dd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-w3-1795dd", "reconcileID": "d2170881-2d26-4585-8e15-6adce9e11e50", "Gateway": "e2e-shared-a-w3-1795dd", "namespace": "openshift-ingress", "name": "e2e-shared-a-w3-1795dd-authn-ssl"} 2026-09-07T17:25:58Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-w3-1795dd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-w3-1795dd", "reconcileID": "d2170881-2d26-4585-8e15-6adce9e11e50", "Gateway": "e2e-shared-a-w3-1795dd", "namespace": "openshift-ingress"} 2026-09-07T17:26:04Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-w3-1795dd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-w3-1795dd", "reconcileID": "cdf83dd8-4f1b-4593-b40e-82b661180e0f", "Gateway": "e2e-shared-a-w3-1795dd", "namespace": "openshift-ingress"} 2026-09-07T17:26:04Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-w3-1795dd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-w3-1795dd", "reconcileID": "cdf83dd8-4f1b-4593-b40e-82b661180e0f", "Gateway": "e2e-shared-a-w3-1795dd", "namespace": "openshift-ingress", "name": "e2e-shared-a-w3-1795dd-authn-ssl"} 2026-09-07T17:26:04Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-w3-1795dd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-w3-1795dd", "reconcileID": "cdf83dd8-4f1b-4593-b40e-82b661180e0f", "Gateway": "e2e-shared-a-w3-1795dd", "namespace": "openshift-ingress"} 2026-09-07T17:26:05Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-w3-1795dd"}, "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "eea3b7be-c5cd-4b3f-9ff7-3e6b06f479b9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-w3-1795dd"} 2026-09-07T17:26:05Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-w3-1795dd"}, "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "eea3b7be-c5cd-4b3f-9ff7-3e6b06f479b9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "create": "odh-kserve-custom-ca-bundle"} 2026-09-07T17:26:05Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-w3-1795dd"}, "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "eea3b7be-c5cd-4b3f-9ff7-3e6b06f479b9", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-w3-1795dd because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-09-07T17:26:05Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-a-w3-1795dd"}, "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "name": "odh-trusted-ca-bundle", "reconcileID": "7093d709-a976-485f-b848-4d6a261b8c28", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-w3-1795dd"} 2026-09-07T17:26:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-a-w3-1795dd"}, "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "name": "odh-trusted-ca-bundle", "reconcileID": "7093d709-a976-485f-b848-4d6a261b8c28", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-w3-1795dd"} 2026-09-07T17:26:05Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-w3-1795dd"}, "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "name": "openshift-service-ca.crt", "reconcileID": "e081bf78-b96a-4ac7-a224-600dd9db86cb", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-a-w3-1795dd"} 2026-09-07T17:26:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-w3-1795dd"}, "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "name": "openshift-service-ca.crt", "reconcileID": "e081bf78-b96a-4ac7-a224-600dd9db86cb", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-a-w3-1795dd"} 2026-09-07T17:26:05Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-w3-1795dd"}, "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "96c9098f-0361-4ed0-9b58-0c723b3dde25", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-w3-1795dd"} 2026-09-07T17:26:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-w3-1795dd"}, "namespace": "ai-tenant-e2e-shared-a-w3-1795dd", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "96c9098f-0361-4ed0-9b58-0c723b3dde25", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-w3-1795dd"} 2026-09-07T17:26:06Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-w3-1795dd","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-w3-1795dd", "reconcileID": "7083b7f3-08a9-4df6-9e1b-6477f894da7d", "Gateway": "e2e-shared-a-w3-1795dd", "namespace": "openshift-ingress"} 2026-09-07T17:26:16Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-w3-3973bf","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-w3-3973bf", "reconcileID": "c1099257-24f6-4949-919e-3a9fbfdd3f88", "Gateway": "e2e-shared-b-w3-3973bf", "namespace": "openshift-ingress"} 2026-09-07T17:26:16Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-w3-3973bf","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-w3-3973bf", "reconcileID": "c1099257-24f6-4949-919e-3a9fbfdd3f88", "Gateway": "e2e-shared-b-w3-3973bf", "namespace": "openshift-ingress", "name": "e2e-shared-b-w3-3973bf-authn-ssl"} 2026-09-07T17:26:16Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-w3-3973bf","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-w3-3973bf", "reconcileID": "c1099257-24f6-4949-919e-3a9fbfdd3f88", "Gateway": "e2e-shared-b-w3-3973bf", "namespace": "openshift-ingress"} 2026-09-07T17:26:19Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-w3-3973bf","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-w3-3973bf", "reconcileID": "1d4931a6-2c6c-470b-a280-ef746e3c8371", "Gateway": "e2e-shared-b-w3-3973bf", "namespace": "openshift-ingress"} 2026-09-07T17:26:19Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-w3-3973bf","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-w3-3973bf", "reconcileID": "1d4931a6-2c6c-470b-a280-ef746e3c8371", "Gateway": "e2e-shared-b-w3-3973bf", "namespace": "openshift-ingress", "name": "e2e-shared-b-w3-3973bf-authn-ssl"} 2026-09-07T17:26:19Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-w3-3973bf","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-w3-3973bf", "reconcileID": "1d4931a6-2c6c-470b-a280-ef746e3c8371", "Gateway": "e2e-shared-b-w3-3973bf", "namespace": "openshift-ingress"} 2026-09-07T17:26:23Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "acf1fd7a-d288-46ff-ad5e-dbeaf3600799", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf"} 2026-09-07T17:26:23Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "acf1fd7a-d288-46ff-ad5e-dbeaf3600799", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "create": "odh-kserve-custom-ca-bundle"} 2026-09-07T17:26:23Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "acf1fd7a-d288-46ff-ad5e-dbeaf3600799", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-w3-3973bf because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-09-07T17:26:23Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "odh-trusted-ca-bundle", "reconcileID": "1c9493cb-4204-4c79-a5ee-fb62e5a2390f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf"} 2026-09-07T17:26:23Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "odh-trusted-ca-bundle", "reconcileID": "1c9493cb-4204-4c79-a5ee-fb62e5a2390f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf"} 2026-09-07T17:26:23Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "openshift-service-ca.crt", "reconcileID": "5e8b7578-a88d-46ed-8b4e-132028b66b26", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf"} 2026-09-07T17:26:23Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "openshift-service-ca.crt", "reconcileID": "5e8b7578-a88d-46ed-8b4e-132028b66b26", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf"} 2026-09-07T17:26:23Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c14680f9-b399-48f2-8217-4268c1f3fa3c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf"} 2026-09-07T17:26:23Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-w3-3973bf"}, "namespace": "ai-tenant-e2e-shared-b-w3-3973bf", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c14680f9-b399-48f2-8217-4268c1f3fa3c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-w3-3973bf"} 2026-09-07T17:26:24Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-w3-3973bf","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-w3-3973bf", "reconcileID": "8f32884d-a1ac-446f-855e-3c3ece0b426b", "Gateway": "e2e-shared-b-w3-3973bf", "namespace": "openshift-ingress"} 2026-09-07T17:30:07Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-09-07T17:30:07Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-09-07T17:30:10Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-09-07T17:30:10Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-09-07T17:30:45Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "a035aa4a-660e-484e-bb4d-6eec5d196b88", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm"} 2026-09-07T17:30:45Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "a035aa4a-660e-484e-bb4d-6eec5d196b88", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm"} 2026-09-07T17:30:45Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "a035aa4a-660e-484e-bb4d-6eec5d196b88", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm"} 2026-09-07T17:30:45Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "93266b71-63cb-4588-9305-19af0ee7f698", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-09-07T17:30:45Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "a035aa4a-660e-484e-bb4d-6eec5d196b88", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm", "name": "e2e-same-id-a-267c53da-kserve-route-authn"} 2026-09-07T17:30:45Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "a035aa4a-660e-484e-bb4d-6eec5d196b88", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm", "authType": "userdefined"} 2026-09-07T17:30:45Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "a035aa4a-660e-484e-bb4d-6eec5d196b88", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm"} 2026-09-07T17:30:45Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "93266b71-63cb-4588-9305-19af0ee7f698", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-09-07T17:30:45Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "93266b71-63cb-4588-9305-19af0ee7f698", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-09-07T17:30:46Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "d9d8a76a-20f4-461a-9d47-bfc1f4ec1544", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm"} 2026-09-07T17:30:46Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "d9d8a76a-20f4-461a-9d47-bfc1f4ec1544", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm"} 2026-09-07T17:30:46Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "d9d8a76a-20f4-461a-9d47-bfc1f4ec1544", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm"} 2026-09-07T17:30:46Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "1f04affc-888d-4bd1-8052-5bd8b8e3e61b", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-09-07T17:30:46Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "d9d8a76a-20f4-461a-9d47-bfc1f4ec1544", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm", "name": "e2e-same-id-b-267c53da-kserve-route-authn"} 2026-09-07T17:30:46Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "d9d8a76a-20f4-461a-9d47-bfc1f4ec1544", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm", "authType": "userdefined"} 2026-09-07T17:30:46Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "d9d8a76a-20f4-461a-9d47-bfc1f4ec1544", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm"} 2026-09-07T17:30:46Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "1f04affc-888d-4bd1-8052-5bd8b8e3e61b", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-09-07T17:30:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "1f04affc-888d-4bd1-8052-5bd8b8e3e61b", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-09-07T17:30:46Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "29d4f011-2caf-4786-bb2e-a975bbf12caa", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm"} 2026-09-07T17:30:46Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "29d4f011-2caf-4786-bb2e-a975bbf12caa", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm"} 2026-09-07T17:30:46Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "29d4f011-2caf-4786-bb2e-a975bbf12caa", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm"} 2026-09-07T17:30:46Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "29d4f011-2caf-4786-bb2e-a975bbf12caa", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm", "name": "e2e-same-id-a-267c53da-kserve-route-authn"} 2026-09-07T17:30:46Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "29d4f011-2caf-4786-bb2e-a975bbf12caa", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm", "authType": "userdefined"} 2026-09-07T17:30:46Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "29d4f011-2caf-4786-bb2e-a975bbf12caa", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm"} 2026-09-07T17:31:05Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "afb33a33-63e7-42d7-a00e-5803966a9047", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-09-07T17:31:05Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "afb33a33-63e7-42d7-a00e-5803966a9047", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-09-07T17:31:05Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "afb33a33-63e7-42d7-a00e-5803966a9047", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-09-07T17:31:05Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "7a4e4096-79fd-4456-b392-b60b4708b414", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm"} 2026-09-07T17:31:05Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "7a4e4096-79fd-4456-b392-b60b4708b414", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm"} 2026-09-07T17:31:05Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "7a4e4096-79fd-4456-b392-b60b4708b414", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm"} 2026-09-07T17:31:05Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "7a4e4096-79fd-4456-b392-b60b4708b414", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm", "name": "e2e-same-id-a-267c53da-kserve-route-authn"} 2026-09-07T17:31:05Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "7a4e4096-79fd-4456-b392-b60b4708b414", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm", "authType": "userdefined"} 2026-09-07T17:31:05Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "7a4e4096-79fd-4456-b392-b60b4708b414", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm"} 2026-09-07T17:31:06Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "6fd8681f-09de-4d78-982c-b195ed195700", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm"} 2026-09-07T17:31:06Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "6fd8681f-09de-4d78-982c-b195ed195700", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm"} 2026-09-07T17:31:06Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "6fd8681f-09de-4d78-982c-b195ed195700", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm"} 2026-09-07T17:31:06Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "6fd8681f-09de-4d78-982c-b195ed195700", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm", "name": "e2e-same-id-b-267c53da-kserve-route-authn"} 2026-09-07T17:31:06Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "6fd8681f-09de-4d78-982c-b195ed195700", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm", "authType": "userdefined"} 2026-09-07T17:31:06Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "6fd8681f-09de-4d78-982c-b195ed195700", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm"} 2026-09-07T17:31:18Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "ccf75c6d-6083-4c71-be47-f37a8ab72800", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-09-07T17:31:18Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "ccf75c6d-6083-4c71-be47-f37a8ab72800", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-09-07T17:31:18Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "ccf75c6d-6083-4c71-be47-f37a8ab72800", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-09-07T17:31:19Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "7bf3f8ea-61ba-4456-a6f4-3ab420a357f1", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm"} 2026-09-07T17:31:19Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "7bf3f8ea-61ba-4456-a6f4-3ab420a357f1", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm"} 2026-09-07T17:31:19Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "7bf3f8ea-61ba-4456-a6f4-3ab420a357f1", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm"} 2026-09-07T17:31:19Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "7bf3f8ea-61ba-4456-a6f4-3ab420a357f1", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm", "name": "e2e-same-id-b-267c53da-kserve-route-authn"} 2026-09-07T17:31:19Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "7bf3f8ea-61ba-4456-a6f4-3ab420a357f1", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm", "authType": "userdefined"} 2026-09-07T17:31:19Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "7bf3f8ea-61ba-4456-a6f4-3ab420a357f1", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm"} 2026-09-07T17:31:39Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "b5b40916-fcd3-4730-a92b-2357f407165f", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm"} 2026-09-07T17:31:39Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "b5b40916-fcd3-4730-a92b-2357f407165f", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm"} 2026-09-07T17:31:39Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "b5b40916-fcd3-4730-a92b-2357f407165f", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm"} 2026-09-07T17:31:39Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "b5b40916-fcd3-4730-a92b-2357f407165f", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm", "name": "e2e-same-id-a-267c53da-kserve-route-authn"} 2026-09-07T17:31:39Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "b5b40916-fcd3-4730-a92b-2357f407165f", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm", "authType": "userdefined"} 2026-09-07T17:31:39Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "b5b40916-fcd3-4730-a92b-2357f407165f", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm"} 2026-09-07T17:31:56Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "02a1d729-9d4e-4f17-ae79-7feb6e4a861c", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm"} 2026-09-07T17:31:56Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "02a1d729-9d4e-4f17-ae79-7feb6e4a861c", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm"} 2026-09-07T17:31:56Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "02a1d729-9d4e-4f17-ae79-7feb6e4a861c", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm"} 2026-09-07T17:31:56Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "02a1d729-9d4e-4f17-ae79-7feb6e4a861c", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm", "name": "e2e-same-id-b-267c53da-kserve-route-authn"} 2026-09-07T17:31:56Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "02a1d729-9d4e-4f17-ae79-7feb6e4a861c", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm", "authType": "userdefined"} 2026-09-07T17:31:56Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "02a1d729-9d4e-4f17-ae79-7feb6e4a861c", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm"} 2026-09-07T17:31:58Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "a08d89ee-cbfb-47d2-8b94-60e7885a6908", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm"} 2026-09-07T17:31:58Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "a08d89ee-cbfb-47d2-8b94-60e7885a6908", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm", "name": "e2e-same-id-a-267c53da"} 2026-09-07T17:31:58Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "a08d89ee-cbfb-47d2-8b94-60e7885a6908", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm"} 2026-09-07T17:31:58Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "a08d89ee-cbfb-47d2-8b94-60e7885a6908", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm"} 2026-09-07T17:31:58Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "a08d89ee-cbfb-47d2-8b94-60e7885a6908", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm", "name": "e2e-same-id-a-267c53da-kserve-route-authn"} 2026-09-07T17:31:58Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "a08d89ee-cbfb-47d2-8b94-60e7885a6908", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm", "name": "e2e-same-id-a-267c53da-kserve-route-authn"} 2026-09-07T17:31:58Z INFO Stop LLMInferenceService reconciliation {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-a-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-a-267c53da", "reconcileID": "9f423f4b-f78f-40c9-aeef-49825390f41d", "LLMInferenceService": "e2e-same-id-a-267c53da", "namespace": "llm"} 2026-09-07T17:31:58Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0c72e4d0-81f6-4d12-a934-488a49acc6e5", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-09-07T17:31:58Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0c72e4d0-81f6-4d12-a934-488a49acc6e5", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-09-07T17:31:58Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0c72e4d0-81f6-4d12-a934-488a49acc6e5", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-09-07T17:31:58Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "c5241902-92dc-42b1-9a95-1b0dec4fddb8", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-09-07T17:31:58Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "c5241902-92dc-42b1-9a95-1b0dec4fddb8", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-09-07T17:31:58Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "c5241902-92dc-42b1-9a95-1b0dec4fddb8", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-09-07T17:31:58Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "837709a3-8839-4d6a-b19e-9ab883f0686c", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm"} 2026-09-07T17:31:58Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "837709a3-8839-4d6a-b19e-9ab883f0686c", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm", "name": "e2e-same-id-b-267c53da"} 2026-09-07T17:31:58Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "837709a3-8839-4d6a-b19e-9ab883f0686c", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm"} 2026-09-07T17:31:58Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "837709a3-8839-4d6a-b19e-9ab883f0686c", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm"} 2026-09-07T17:31:58Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "837709a3-8839-4d6a-b19e-9ab883f0686c", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm", "name": "e2e-same-id-b-267c53da-kserve-route-authn"} 2026-09-07T17:31:58Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "837709a3-8839-4d6a-b19e-9ab883f0686c", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm", "name": "e2e-same-id-b-267c53da-kserve-route-authn"} 2026-09-07T17:32:00Z INFO Stop LLMInferenceService reconciliation {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-same-id-b-267c53da","namespace":"llm"}, "namespace": "llm", "name": "e2e-same-id-b-267c53da", "reconcileID": "4635dc67-0fbd-425e-aab0-3edb4712457a", "LLMInferenceService": "e2e-same-id-b-267c53da", "namespace": "llm"} 2026-09-07T17:32:00Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "81123622-e067-476c-9026-dd083c983007", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-09-07T17:32:00Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "81123622-e067-476c-9026-dd083c983007", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-09-07T17:32:00Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "81123622-e067-476c-9026-dd083c983007", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-09-07T17:32:00Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "a4609f1c-ac67-4e6e-92a3-2501b479c204", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-09-07T17:32:00Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "a4609f1c-ac67-4e6e-92a3-2501b479c204", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-09-07T17:32:00Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "a4609f1c-ac67-4e6e-92a3-2501b479c204", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"}