2026-07-08T03:35:44Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-7142f2","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "test-model-7142f2", "reconcileID": "d81b464a-4e10-41a6-b0c9-0f9a3faa64eb", "LLMInferenceService": "test-model-7142f2", "namespace": "ai-tenant-e2e-inf-b-7142f2"} 2026-07-08T03:35:48Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-7142f2","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "test-model-7142f2", "reconcileID": "2ec2037c-6e1a-43f4-b755-c39a9beb065f", "LLMInferenceService": "test-model-7142f2", "namespace": "ai-tenant-e2e-inf-b-7142f2"} 2026-07-08T03:35:48Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-7142f2","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "test-model-7142f2", "reconcileID": "2ec2037c-6e1a-43f4-b755-c39a9beb065f", "LLMInferenceService": "test-model-7142f2", "namespace": "ai-tenant-e2e-inf-b-7142f2"} 2026-07-08T03:35:48Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-7142f2","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "test-model-7142f2", "reconcileID": "2ec2037c-6e1a-43f4-b755-c39a9beb065f", "LLMInferenceService": "test-model-7142f2", "namespace": "ai-tenant-e2e-inf-b-7142f2"} 2026-07-08T03:35:48Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-7142f2","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "test-model-7142f2", "reconcileID": "2ec2037c-6e1a-43f4-b755-c39a9beb065f", "LLMInferenceService": "test-model-7142f2", "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "test-model-7142f2-kserve-route-authn"} 2026-07-08T03:35:48Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-7142f2","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "test-model-7142f2", "reconcileID": "2ec2037c-6e1a-43f4-b755-c39a9beb065f", "LLMInferenceService": "test-model-7142f2", "namespace": "ai-tenant-e2e-inf-b-7142f2", "authType": "userdefined"} 2026-07-08T03:35:48Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-7142f2","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "test-model-7142f2", "reconcileID": "2ec2037c-6e1a-43f4-b755-c39a9beb065f", "LLMInferenceService": "test-model-7142f2", "namespace": "ai-tenant-e2e-inf-b-7142f2"} 2026-07-08T03:35:50Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-7142f2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-7142f2", "reconcileID": "6834738b-f54b-45d3-8bae-dffb6c1214e5", "Gateway": "e2e-inf-b-7142f2", "namespace": "openshift-ingress"} 2026-07-08T03:35:50Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-7142f2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-7142f2", "reconcileID": "6834738b-f54b-45d3-8bae-dffb6c1214e5", "Gateway": "e2e-inf-b-7142f2", "namespace": "openshift-ingress", "name": "e2e-inf-b-7142f2-authn-ssl"} 2026-07-08T03:35:50Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-7142f2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-7142f2", "reconcileID": "6834738b-f54b-45d3-8bae-dffb6c1214e5", "Gateway": "e2e-inf-b-7142f2", "namespace": "openshift-ingress"} 2026-07-08T03:35:56Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "cb640f1c-55f2-4a7b-8b1e-ca4df280f507", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:35:56Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "cb640f1c-55f2-4a7b-8b1e-ca4df280f507", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress", "name": "e2e-reuse-25332d-authn-ssl"} 2026-07-08T03:35:56Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "cb640f1c-55f2-4a7b-8b1e-ca4df280f507", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:35:56Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "b7d116c5-0adf-407a-a9b8-1bd95094aa23", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:35:56Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "b7d116c5-0adf-407a-a9b8-1bd95094aa23", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress", "name": "e2e-reuse-25332d-authn-ssl"} 2026-07-08T03:35:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "b7d116c5-0adf-407a-a9b8-1bd95094aa23", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:35:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "715da450-9942-4b39-b684-9688607344c4", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:35:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "715da450-9942-4b39-b684-9688607344c4", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress", "name": "e2e-reuse-25332d-authn-ssl"} 2026-07-08T03:35:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "715da450-9942-4b39-b684-9688607344c4", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:35:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "fa8def93-5d05-4666-b56b-7e915772dc78", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:35:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "fa8def93-5d05-4666-b56b-7e915772dc78", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress", "name": "e2e-reuse-25332d-authn-ssl"} 2026-07-08T03:35:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "fa8def93-5d05-4666-b56b-7e915772dc78", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:35:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "e9abafee-e609-4662-9d79-606243762448", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:35:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "e9abafee-e609-4662-9d79-606243762448", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress", "name": "e2e-reuse-25332d-authn-ssl"} 2026-07-08T03:35:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "e9abafee-e609-4662-9d79-606243762448", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:35:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "c2720e47-a84a-480f-85f6-b01be4ddc689", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:35:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "c2720e47-a84a-480f-85f6-b01be4ddc689", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress", "name": "e2e-reuse-25332d-authn-ssl"} 2026-07-08T03:35:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "c2720e47-a84a-480f-85f6-b01be4ddc689", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:35:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "114e071c-9c5d-40f9-be68-d2a7223fdbaf", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:35:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "114e071c-9c5d-40f9-be68-d2a7223fdbaf", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress", "name": "e2e-reuse-25332d-authn-ssl"} 2026-07-08T03:35:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "114e071c-9c5d-40f9-be68-d2a7223fdbaf", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:35:57Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "1f4fb7fe-6699-4765-b880-73f1a73f0d2f", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:35:57Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "1f4fb7fe-6699-4765-b880-73f1a73f0d2f", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress", "name": "e2e-reuse-25332d-authn-ssl"} 2026-07-08T03:35:57Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "1f4fb7fe-6699-4765-b880-73f1a73f0d2f", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:35:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "openshift-service-ca.crt", "reconcileID": "454af0b1-8b66-4c92-92c4-4aee5eb7f0fb", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:35:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "odh-trusted-ca-bundle", "reconcileID": "ef1a8aa3-2d37-44ba-b71b-fa42066d682b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:35:59Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "openshift-service-ca.crt", "reconcileID": "dbaf704b-3fd9-4025-866a-8830299e3c0c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-reuse-25332d", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T03:35:59Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "openshift-service-ca.crt", "reconcileID": "dbaf704b-3fd9-4025-866a-8830299e3c0c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:35:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "openshift-service-ca.crt", "reconcileID": "4424353a-260a-4d09-b88e-f298a81c4836", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:35:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2e4ca3a0-2bca-422d-b03b-f4e52e6e2a07", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:35:59Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "odh-trusted-ca-bundle", "reconcileID": "69f60c2a-ff4e-4767-8153-e5fd68e58f01", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-reuse-25332d", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T03:35:59Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "odh-trusted-ca-bundle", "reconcileID": "69f60c2a-ff4e-4767-8153-e5fd68e58f01", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:35:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b42f8de4-1990-4816-9615-d52e43c239e6", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:36:00Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "03bb8831-d03d-4cad-93f9-d5097c412504", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:36:00Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "03bb8831-d03d-4cad-93f9-d5097c412504", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress", "name": "e2e-reuse-25332d-authn-ssl"} 2026-07-08T03:36:00Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "03bb8831-d03d-4cad-93f9-d5097c412504", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:36:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "2b671ab4-cfd7-4c0d-927b-58bc7b626dac", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T03:36:04Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "9d2ba5bb-4ba8-40c9-892e-059f0372bdbe", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:36:04Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "9d2ba5bb-4ba8-40c9-892e-059f0372bdbe", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:36:04Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "9d2ba5bb-4ba8-40c9-892e-059f0372bdbe", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:36:04Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "f5ae7951-73b5-4ac0-95c1-928e922a5238", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:36:04Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "9d2ba5bb-4ba8-40c9-892e-059f0372bdbe", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d-kserve-route-authn"} 2026-07-08T03:36:04Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "9d2ba5bb-4ba8-40c9-892e-059f0372bdbe", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "authType": "userdefined"} 2026-07-08T03:36:04Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "9d2ba5bb-4ba8-40c9-892e-059f0372bdbe", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:36:04Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "f5ae7951-73b5-4ac0-95c1-928e922a5238", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress", "name": "e2e-reuse-25332d-authn-ssl"} 2026-07-08T03:36:04Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "f5ae7951-73b5-4ac0-95c1-928e922a5238", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:36:04Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "f80178ce-aaf4-43b5-8ea3-8a1ecb88ab51", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:36:04Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "f80178ce-aaf4-43b5-8ea3-8a1ecb88ab51", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:36:04Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "f80178ce-aaf4-43b5-8ea3-8a1ecb88ab51", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:36:04Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "f80178ce-aaf4-43b5-8ea3-8a1ecb88ab51", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d-kserve-route-authn"} 2026-07-08T03:36:04Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "f80178ce-aaf4-43b5-8ea3-8a1ecb88ab51", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "authType": "userdefined"} 2026-07-08T03:36:04Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "f80178ce-aaf4-43b5-8ea3-8a1ecb88ab51", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:36:07Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "742aecc3-fb2e-4a8f-b22d-5c2ca3fb2b92", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:36:07Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "742aecc3-fb2e-4a8f-b22d-5c2ca3fb2b92", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress", "name": "e2e-reuse-25332d-authn-ssl"} 2026-07-08T03:36:07Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "742aecc3-fb2e-4a8f-b22d-5c2ca3fb2b92", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:36:33Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "ba5c4311-2c01-49d2-a80e-0080d72bf5bc", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:36:33Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "ba5c4311-2c01-49d2-a80e-0080d72bf5bc", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress", "name": "e2e-reuse-25332d-authn-ssl"} 2026-07-08T03:36:33Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "ba5c4311-2c01-49d2-a80e-0080d72bf5bc", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:36:34Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "1c4df486-d048-4c00-b8fb-c27eaed7bcc9", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:36:34Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "1c4df486-d048-4c00-b8fb-c27eaed7bcc9", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:36:34Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "1c4df486-d048-4c00-b8fb-c27eaed7bcc9", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:36:34Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "1c4df486-d048-4c00-b8fb-c27eaed7bcc9", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d-kserve-route-authn"} 2026-07-08T03:36:34Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "1c4df486-d048-4c00-b8fb-c27eaed7bcc9", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "authType": "userdefined"} 2026-07-08T03:36:34Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "1c4df486-d048-4c00-b8fb-c27eaed7bcc9", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:36:43Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "8c5875ba-ff93-4c89-996a-57d7f9afbe4f", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:36:43Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "8c5875ba-ff93-4c89-996a-57d7f9afbe4f", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress", "name": "e2e-reuse-25332d-authn-ssl"} 2026-07-08T03:36:43Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "8c5875ba-ff93-4c89-996a-57d7f9afbe4f", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:36:45Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "0adf1617-253e-4a4b-8269-daa7f4c16495", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:36:45Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "0adf1617-253e-4a4b-8269-daa7f4c16495", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:36:45Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "0adf1617-253e-4a4b-8269-daa7f4c16495", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:36:45Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "0adf1617-253e-4a4b-8269-daa7f4c16495", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d-kserve-route-authn"} 2026-07-08T03:36:45Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "0adf1617-253e-4a4b-8269-daa7f4c16495", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "authType": "userdefined"} 2026-07-08T03:36:45Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "0adf1617-253e-4a4b-8269-daa7f4c16495", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:36:46Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "5294d19c-35b9-4111-8e65-cdec79858148", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:36:46Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "5294d19c-35b9-4111-8e65-cdec79858148", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress", "name": "e2e-reuse-25332d-authn-ssl"} 2026-07-08T03:36:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "5294d19c-35b9-4111-8e65-cdec79858148", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:36:46Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "1762aab5-304f-4bbd-891a-d3311b96f702", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:36:46Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "1762aab5-304f-4bbd-891a-d3311b96f702", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress", "name": "e2e-reuse-25332d-authn-ssl"} 2026-07-08T03:36:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "1762aab5-304f-4bbd-891a-d3311b96f702", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:36:52Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "25ffcdf0-4dd5-45b2-ad9f-4fb4b408d7b6", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:36:52Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "25ffcdf0-4dd5-45b2-ad9f-4fb4b408d7b6", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:36:52Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "25ffcdf0-4dd5-45b2-ad9f-4fb4b408d7b6", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:36:52Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "25ffcdf0-4dd5-45b2-ad9f-4fb4b408d7b6", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d-kserve-route-authn"} 2026-07-08T03:36:52Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "25ffcdf0-4dd5-45b2-ad9f-4fb4b408d7b6", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "authType": "userdefined"} 2026-07-08T03:36:52Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "25ffcdf0-4dd5-45b2-ad9f-4fb4b408d7b6", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "1618fdd8-5ad1-4829-a81d-fe8210de716f", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "1618fdd8-5ad1-4829-a81d-fe8210de716f", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "1618fdd8-5ad1-4829-a81d-fe8210de716f", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "1618fdd8-5ad1-4829-a81d-fe8210de716f", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d-kserve-route-authn"} 2026-07-08T03:37:28Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "1618fdd8-5ad1-4829-a81d-fe8210de716f", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "authType": "userdefined"} 2026-07-08T03:37:28Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "1618fdd8-5ad1-4829-a81d-fe8210de716f", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "c7220e1b-522f-4206-8760-4d4f028ab9ac", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "c7220e1b-522f-4206-8760-4d4f028ab9ac", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "c7220e1b-522f-4206-8760-4d4f028ab9ac", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "c7220e1b-522f-4206-8760-4d4f028ab9ac", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d-kserve-route-authn"} 2026-07-08T03:37:28Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "c7220e1b-522f-4206-8760-4d4f028ab9ac", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "authType": "userdefined"} 2026-07-08T03:37:28Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "c7220e1b-522f-4206-8760-4d4f028ab9ac", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "bb538648-5ffc-4491-8e1f-1e786b891279", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "bb538648-5ffc-4491-8e1f-1e786b891279", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "bb538648-5ffc-4491-8e1f-1e786b891279", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "bb538648-5ffc-4491-8e1f-1e786b891279", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d-kserve-route-authn"} 2026-07-08T03:37:28Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "bb538648-5ffc-4491-8e1f-1e786b891279", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "authType": "userdefined"} 2026-07-08T03:37:28Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "bb538648-5ffc-4491-8e1f-1e786b891279", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "3383e3f1-01b8-42d0-a9f5-02d525a70d95", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "3383e3f1-01b8-42d0-a9f5-02d525a70d95", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "3383e3f1-01b8-42d0-a9f5-02d525a70d95", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "3383e3f1-01b8-42d0-a9f5-02d525a70d95", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d-kserve-route-authn"} 2026-07-08T03:37:28Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "3383e3f1-01b8-42d0-a9f5-02d525a70d95", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "authType": "userdefined"} 2026-07-08T03:37:28Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "3383e3f1-01b8-42d0-a9f5-02d525a70d95", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "e7f8c06d-55ff-4f27-b959-2559190fba6b", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "e7f8c06d-55ff-4f27-b959-2559190fba6b", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d"} 2026-07-08T03:37:28Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "e7f8c06d-55ff-4f27-b959-2559190fba6b", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "e7f8c06d-55ff-4f27-b959-2559190fba6b", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "e7f8c06d-55ff-4f27-b959-2559190fba6b", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d-kserve-route-authn"} 2026-07-08T03:37:28Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "e7f8c06d-55ff-4f27-b959-2559190fba6b", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d-kserve-route-authn"} 2026-07-08T03:37:28Z DEBUG Triggering LLMInferenceService Cleanup for Namespace {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "e7f8c06d-55ff-4f27-b959-2559190fba6b", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Cleaning up LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "e7f8c06d-55ff-4f27-b959-2559190fba6b", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "395e8b45-e0b5-4e89-8b84-7bc689335e1d", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "395e8b45-e0b5-4e89-8b84-7bc689335e1d", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d"} 2026-07-08T03:37:28Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "395e8b45-e0b5-4e89-8b84-7bc689335e1d", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "395e8b45-e0b5-4e89-8b84-7bc689335e1d", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "395e8b45-e0b5-4e89-8b84-7bc689335e1d", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d-kserve-route-authn"} 2026-07-08T03:37:28Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "395e8b45-e0b5-4e89-8b84-7bc689335e1d", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d-kserve-route-authn"} 2026-07-08T03:37:28Z DEBUG Triggering LLMInferenceService Cleanup for Namespace {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "395e8b45-e0b5-4e89-8b84-7bc689335e1d", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:28Z DEBUG Cleaning up LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "395e8b45-e0b5-4e89-8b84-7bc689335e1d", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:29Z INFO Stop LLMInferenceService reconciliation {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-25332d","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "test-model-25332d", "reconcileID": "2f481006-df93-478a-82aa-783bb1cf2bc3", "LLMInferenceService": "test-model-25332d", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:29Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "a404aa15-30ca-4652-8363-9365165068d6", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:37:29Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "a404aa15-30ca-4652-8363-9365165068d6", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress", "name": "e2e-reuse-25332d-authn-ssl"} 2026-07-08T03:37:29Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "a404aa15-30ca-4652-8363-9365165068d6", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:37:29Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "84e7cffc-31e6-4638-831c-f81ecc361041", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:37:29Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "84e7cffc-31e6-4638-831c-f81ecc361041", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress", "name": "e2e-reuse-25332d-authn-ssl"} 2026-07-08T03:37:29Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "84e7cffc-31e6-4638-831c-f81ecc361041", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:37:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e7897aad-ff60-48b2-8071-5ff9b5ad80ce", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:29Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e7897aad-ff60-48b2-8071-5ff9b5ad80ce", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-reuse-25332d", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T03:37:29Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e7897aad-ff60-48b2-8071-5ff9b5ad80ce", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-reuse-25332d because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T03:37:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "odh-trusted-ca-bundle", "reconcileID": "2c46b6a2-c3e8-46b7-a306-e95d5f45cf50", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "odh-trusted-ca-bundle", "reconcileID": "2c46b6a2-c3e8-46b7-a306-e95d5f45cf50", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "openshift-service-ca.crt", "reconcileID": "25f7d18f-ebe2-4076-a52d-c5e541451227", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "openshift-service-ca.crt", "reconcileID": "25f7d18f-ebe2-4076-a52d-c5e541451227", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:29Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dd3cfced-e39c-4b14-aa6c-792a789cc8c1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:29Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-reuse-25332d"}, "namespace": "ai-tenant-e2e-reuse-25332d", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dd3cfced-e39c-4b14-aa6c-792a789cc8c1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-reuse-25332d"} 2026-07-08T03:37:35Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-reuse-25332d","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-reuse-25332d", "reconcileID": "7d265216-f334-443f-b29f-dd8f37e4f933", "Gateway": "e2e-reuse-25332d", "namespace": "openshift-ingress"} 2026-07-08T03:37:36Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-2515cc","namespace":"ai-tenant-e2e-inf-a-2515cc"}, "namespace": "ai-tenant-e2e-inf-a-2515cc", "name": "test-model-2515cc", "reconcileID": "13d20bff-4ad0-479d-89e6-c4f6a3cbce8e", "LLMInferenceService": "test-model-2515cc", "namespace": "ai-tenant-e2e-inf-a-2515cc"} 2026-07-08T03:37:36Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-2515cc","namespace":"ai-tenant-e2e-inf-a-2515cc"}, "namespace": "ai-tenant-e2e-inf-a-2515cc", "name": "test-model-2515cc", "reconcileID": "13d20bff-4ad0-479d-89e6-c4f6a3cbce8e", "LLMInferenceService": "test-model-2515cc", "namespace": "ai-tenant-e2e-inf-a-2515cc", "name": "test-model-2515cc"} 2026-07-08T03:37:36Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-2515cc","namespace":"ai-tenant-e2e-inf-a-2515cc"}, "namespace": "ai-tenant-e2e-inf-a-2515cc", "name": "test-model-2515cc", "reconcileID": "13d20bff-4ad0-479d-89e6-c4f6a3cbce8e", "LLMInferenceService": "test-model-2515cc", "namespace": "ai-tenant-e2e-inf-a-2515cc"} 2026-07-08T03:37:36Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-2515cc","namespace":"ai-tenant-e2e-inf-a-2515cc"}, "namespace": "ai-tenant-e2e-inf-a-2515cc", "name": "test-model-2515cc", "reconcileID": "13d20bff-4ad0-479d-89e6-c4f6a3cbce8e", "LLMInferenceService": "test-model-2515cc", "namespace": "ai-tenant-e2e-inf-a-2515cc"} 2026-07-08T03:37:36Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-2515cc","namespace":"ai-tenant-e2e-inf-a-2515cc"}, "namespace": "ai-tenant-e2e-inf-a-2515cc", "name": "test-model-2515cc", "reconcileID": "13d20bff-4ad0-479d-89e6-c4f6a3cbce8e", "LLMInferenceService": "test-model-2515cc", "namespace": "ai-tenant-e2e-inf-a-2515cc", "name": "test-model-2515cc-kserve-route-authn"} 2026-07-08T03:37:36Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-2515cc","namespace":"ai-tenant-e2e-inf-a-2515cc"}, "namespace": "ai-tenant-e2e-inf-a-2515cc", "name": "test-model-2515cc", "reconcileID": "13d20bff-4ad0-479d-89e6-c4f6a3cbce8e", "LLMInferenceService": "test-model-2515cc", "namespace": "ai-tenant-e2e-inf-a-2515cc", "name": "test-model-2515cc-kserve-route-authn"} 2026-07-08T03:37:36Z DEBUG Triggering LLMInferenceService Cleanup for Namespace {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-2515cc","namespace":"ai-tenant-e2e-inf-a-2515cc"}, "namespace": "ai-tenant-e2e-inf-a-2515cc", "name": "test-model-2515cc", "reconcileID": "13d20bff-4ad0-479d-89e6-c4f6a3cbce8e", "LLMInferenceService": "test-model-2515cc", "namespace": "ai-tenant-e2e-inf-a-2515cc", "namespace": "ai-tenant-e2e-inf-a-2515cc"} 2026-07-08T03:37:36Z DEBUG Cleaning up LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-2515cc","namespace":"ai-tenant-e2e-inf-a-2515cc"}, "namespace": "ai-tenant-e2e-inf-a-2515cc", "name": "test-model-2515cc", "reconcileID": "13d20bff-4ad0-479d-89e6-c4f6a3cbce8e", "LLMInferenceService": "test-model-2515cc", "namespace": "ai-tenant-e2e-inf-a-2515cc", "namespace": "ai-tenant-e2e-inf-a-2515cc"} 2026-07-08T03:37:36Z INFO Stop LLMInferenceService reconciliation {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-2515cc","namespace":"ai-tenant-e2e-inf-a-2515cc"}, "namespace": "ai-tenant-e2e-inf-a-2515cc", "name": "test-model-2515cc", "reconcileID": "a7f3886e-a9ef-4946-bc62-ca091073c92b", "LLMInferenceService": "test-model-2515cc", "namespace": "ai-tenant-e2e-inf-a-2515cc"} 2026-07-08T03:37:36Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-2515cc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-2515cc", "reconcileID": "48d2ea61-9f58-477b-ac2c-e97275579522", "Gateway": "e2e-inf-a-2515cc", "namespace": "openshift-ingress"} 2026-07-08T03:37:36Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-2515cc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-2515cc", "reconcileID": "48d2ea61-9f58-477b-ac2c-e97275579522", "Gateway": "e2e-inf-a-2515cc", "namespace": "openshift-ingress", "name": "e2e-inf-a-2515cc-authn-ssl"} 2026-07-08T03:37:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-2515cc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-2515cc", "reconcileID": "48d2ea61-9f58-477b-ac2c-e97275579522", "Gateway": "e2e-inf-a-2515cc", "namespace": "openshift-ingress"} 2026-07-08T03:37:36Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-2515cc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-2515cc", "reconcileID": "36ab45c2-06a9-4728-a750-9be50fa674b5", "Gateway": "e2e-inf-a-2515cc", "namespace": "openshift-ingress"} 2026-07-08T03:37:36Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-2515cc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-2515cc", "reconcileID": "36ab45c2-06a9-4728-a750-9be50fa674b5", "Gateway": "e2e-inf-a-2515cc", "namespace": "openshift-ingress", "name": "e2e-inf-a-2515cc-authn-ssl"} 2026-07-08T03:37:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-2515cc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-2515cc", "reconcileID": "36ab45c2-06a9-4728-a750-9be50fa674b5", "Gateway": "e2e-inf-a-2515cc", "namespace": "openshift-ingress"} 2026-07-08T03:37:37Z INFO LLMInferenceService being deleted, cleaning up sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-7142f2","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "test-model-7142f2", "reconcileID": "16cda701-644a-43b6-96df-7a30a5d48116", "LLMInferenceService": "test-model-7142f2", "namespace": "ai-tenant-e2e-inf-b-7142f2"} 2026-07-08T03:37:37Z DEBUG Triggering Delete for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-7142f2","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "test-model-7142f2", "reconcileID": "16cda701-644a-43b6-96df-7a30a5d48116", "LLMInferenceService": "test-model-7142f2", "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "test-model-7142f2"} 2026-07-08T03:37:37Z DEBUG Deleting AuthPolicies for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-7142f2","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "test-model-7142f2", "reconcileID": "16cda701-644a-43b6-96df-7a30a5d48116", "LLMInferenceService": "test-model-7142f2", "namespace": "ai-tenant-e2e-inf-b-7142f2"} 2026-07-08T03:37:37Z DEBUG Deleting HTTPRoute AuthPolicy for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-7142f2","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "test-model-7142f2", "reconcileID": "16cda701-644a-43b6-96df-7a30a5d48116", "LLMInferenceService": "test-model-7142f2", "namespace": "ai-tenant-e2e-inf-b-7142f2"} 2026-07-08T03:37:37Z DEBUG Attempting to delete HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-7142f2","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "test-model-7142f2", "reconcileID": "16cda701-644a-43b6-96df-7a30a5d48116", "LLMInferenceService": "test-model-7142f2", "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "test-model-7142f2-kserve-route-authn"} 2026-07-08T03:37:37Z INFO Successfully deleted HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-7142f2","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "test-model-7142f2", "reconcileID": "16cda701-644a-43b6-96df-7a30a5d48116", "LLMInferenceService": "test-model-7142f2", "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "test-model-7142f2-kserve-route-authn"} 2026-07-08T03:37:37Z DEBUG Triggering LLMInferenceService Cleanup for Namespace {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-7142f2","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "test-model-7142f2", "reconcileID": "16cda701-644a-43b6-96df-7a30a5d48116", "LLMInferenceService": "test-model-7142f2", "namespace": "ai-tenant-e2e-inf-b-7142f2", "namespace": "ai-tenant-e2e-inf-b-7142f2"} 2026-07-08T03:37:37Z DEBUG Cleaning up LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-7142f2","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "test-model-7142f2", "reconcileID": "16cda701-644a-43b6-96df-7a30a5d48116", "LLMInferenceService": "test-model-7142f2", "namespace": "ai-tenant-e2e-inf-b-7142f2", "namespace": "ai-tenant-e2e-inf-b-7142f2"} 2026-07-08T03:37:37Z INFO Stop LLMInferenceService reconciliation {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"test-model-7142f2","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "test-model-7142f2", "reconcileID": "cc7b93e5-7730-4d13-9f19-e33ae8f62bea", "LLMInferenceService": "test-model-7142f2", "namespace": "ai-tenant-e2e-inf-b-7142f2"} 2026-07-08T03:37:37Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-7142f2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-7142f2", "reconcileID": "c30bf193-3616-4dee-b3fe-61207f3d07b4", "Gateway": "e2e-inf-b-7142f2", "namespace": "openshift-ingress"} 2026-07-08T03:37:37Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-7142f2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-7142f2", "reconcileID": "c30bf193-3616-4dee-b3fe-61207f3d07b4", "Gateway": "e2e-inf-b-7142f2", "namespace": "openshift-ingress", "name": "e2e-inf-b-7142f2-authn-ssl"} 2026-07-08T03:37:37Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-7142f2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-7142f2", "reconcileID": "c30bf193-3616-4dee-b3fe-61207f3d07b4", "Gateway": "e2e-inf-b-7142f2", "namespace": "openshift-ingress"} 2026-07-08T03:37:37Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-7142f2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-7142f2", "reconcileID": "284f507d-339c-4609-a671-4f21d2ba0391", "Gateway": "e2e-inf-b-7142f2", "namespace": "openshift-ingress"} 2026-07-08T03:37:37Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-7142f2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-7142f2", "reconcileID": "284f507d-339c-4609-a671-4f21d2ba0391", "Gateway": "e2e-inf-b-7142f2", "namespace": "openshift-ingress", "name": "e2e-inf-b-7142f2-authn-ssl"} 2026-07-08T03:37:37Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-7142f2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-7142f2", "reconcileID": "284f507d-339c-4609-a671-4f21d2ba0391", "Gateway": "e2e-inf-b-7142f2", "namespace": "openshift-ingress"} 2026-07-08T03:37:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "9394dad0-04e7-4796-98a4-156d36c38aaa", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T03:37:38Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-2515cc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-2515cc", "reconcileID": "9508a4b2-3185-4fe5-9ddc-60c73b83555b", "Gateway": "e2e-inf-a-2515cc", "namespace": "openshift-ingress"} 2026-07-08T03:37:38Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-2515cc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-2515cc", "reconcileID": "9508a4b2-3185-4fe5-9ddc-60c73b83555b", "Gateway": "e2e-inf-a-2515cc", "namespace": "openshift-ingress", "name": "e2e-inf-a-2515cc-authn-ssl"} 2026-07-08T03:37:38Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-2515cc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-2515cc", "reconcileID": "9508a4b2-3185-4fe5-9ddc-60c73b83555b", "Gateway": "e2e-inf-a-2515cc", "namespace": "openshift-ingress"} 2026-07-08T03:38:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-2515cc"}, "namespace": "ai-tenant-e2e-inf-a-2515cc", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3cf55862-a469-4172-94b6-be1557c9d914", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-2515cc"} 2026-07-08T03:38:19Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-2515cc"}, "namespace": "ai-tenant-e2e-inf-a-2515cc", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3cf55862-a469-4172-94b6-be1557c9d914", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-2515cc", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T03:38:19Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-2515cc"}, "namespace": "ai-tenant-e2e-inf-a-2515cc", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3cf55862-a469-4172-94b6-be1557c9d914", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-inf-a-2515cc because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T03:38:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-2515cc"}, "namespace": "ai-tenant-e2e-inf-a-2515cc", "name": "odh-trusted-ca-bundle", "reconcileID": "205c63a0-4b83-4866-bc6f-45da92e65559", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-2515cc"} 2026-07-08T03:38:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-a-2515cc"}, "namespace": "ai-tenant-e2e-inf-a-2515cc", "name": "odh-trusted-ca-bundle", "reconcileID": "205c63a0-4b83-4866-bc6f-45da92e65559", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-2515cc"} 2026-07-08T03:38:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-2515cc"}, "namespace": "ai-tenant-e2e-inf-a-2515cc", "name": "openshift-service-ca.crt", "reconcileID": "dc5664ef-1a29-41a1-b7f8-5b70eb2b2aac", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-2515cc"} 2026-07-08T03:38:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-a-2515cc"}, "namespace": "ai-tenant-e2e-inf-a-2515cc", "name": "openshift-service-ca.crt", "reconcileID": "dc5664ef-1a29-41a1-b7f8-5b70eb2b2aac", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-a-2515cc"} 2026-07-08T03:38:19Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-2515cc"}, "namespace": "ai-tenant-e2e-inf-a-2515cc", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8c0f4d6b-8f84-4e14-94e1-b50b4551862f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-2515cc"} 2026-07-08T03:38:19Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-a-2515cc"}, "namespace": "ai-tenant-e2e-inf-a-2515cc", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8c0f4d6b-8f84-4e14-94e1-b50b4551862f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-a-2515cc"} 2026-07-08T03:38:20Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-a-2515cc","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-a-2515cc", "reconcileID": "8aef1ef4-7b0e-409e-b63d-b95c20c7abfe", "Gateway": "e2e-inf-a-2515cc", "namespace": "openshift-ingress"} 2026-07-08T03:38:21Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "4429e8f3-a186-4a45-addf-7402d28c769a", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T03:38:21Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-7142f2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-7142f2", "reconcileID": "e08131ef-2796-4025-9242-5db3289b2530", "Gateway": "e2e-inf-b-7142f2", "namespace": "openshift-ingress"} 2026-07-08T03:38:21Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-7142f2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-7142f2", "reconcileID": "e08131ef-2796-4025-9242-5db3289b2530", "Gateway": "e2e-inf-b-7142f2", "namespace": "openshift-ingress", "name": "e2e-inf-b-7142f2-authn-ssl"} 2026-07-08T03:38:21Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-7142f2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-7142f2", "reconcileID": "e08131ef-2796-4025-9242-5db3289b2530", "Gateway": "e2e-inf-b-7142f2", "namespace": "openshift-ingress"} 2026-07-08T03:38:26Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "886627c9-8c98-4774-912d-2e05ec2d96c1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-7142f2"} 2026-07-08T03:38:26Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "886627c9-8c98-4774-912d-2e05ec2d96c1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-7142f2", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T03:38:26Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "886627c9-8c98-4774-912d-2e05ec2d96c1", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-inf-b-7142f2 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T03:38:26Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "eb69d22b-ee76-4180-8600-19a82a2094fb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-7142f2"} 2026-07-08T03:38:26Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "eb69d22b-ee76-4180-8600-19a82a2094fb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-7142f2", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T03:38:26Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "eb69d22b-ee76-4180-8600-19a82a2094fb", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-inf-b-7142f2 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T03:38:26Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "odh-trusted-ca-bundle", "reconcileID": "a92fa3d3-f474-49c0-8ae7-7bccae74ed87", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-7142f2"} 2026-07-08T03:38:26Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "odh-trusted-ca-bundle", "reconcileID": "a92fa3d3-f474-49c0-8ae7-7bccae74ed87", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-7142f2", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T03:38:26Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "odh-trusted-ca-bundle", "reconcileID": "a92fa3d3-f474-49c0-8ae7-7bccae74ed87", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-inf-b-7142f2 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T03:38:26Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "openshift-service-ca.crt", "reconcileID": "30d6fb39-275f-46cd-bf82-b6556b982dfe", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-7142f2"} 2026-07-08T03:38:26Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "openshift-service-ca.crt", "reconcileID": "30d6fb39-275f-46cd-bf82-b6556b982dfe", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-inf-b-7142f2"} 2026-07-08T03:38:26Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "36ec246f-0836-4654-bc40-99973ca0c8af", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-7142f2"} 2026-07-08T03:38:26Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "36ec246f-0836-4654-bc40-99973ca0c8af", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-7142f2"} 2026-07-08T03:38:26Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "odh-trusted-ca-bundle", "reconcileID": "8d691bd0-a17d-4120-8529-5e9ec6a42a0c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-7142f2"} 2026-07-08T03:38:26Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-inf-b-7142f2"}, "namespace": "ai-tenant-e2e-inf-b-7142f2", "name": "odh-trusted-ca-bundle", "reconcileID": "8d691bd0-a17d-4120-8529-5e9ec6a42a0c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-inf-b-7142f2"} 2026-07-08T03:38:27Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-inf-b-7142f2","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-inf-b-7142f2", "reconcileID": "57c3d544-18b3-44b7-915e-ca5ace96a1e8", "Gateway": "e2e-inf-b-7142f2", "namespace": "openshift-ingress"} 2026-07-08T03:38:28Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "19dcfa36-319d-4a6c-9b50-277299d1e872", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress"} 2026-07-08T03:38:28Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "19dcfa36-319d-4a6c-9b50-277299d1e872", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238-authn-ssl"} 2026-07-08T03:38:28Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "19dcfa36-319d-4a6c-9b50-277299d1e872", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress"} 2026-07-08T03:38:28Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "e3f4cb62-5f7b-4680-a80e-047bf1629a25", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress"} 2026-07-08T03:38:28Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "e3f4cb62-5f7b-4680-a80e-047bf1629a25", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238-authn-ssl"} 2026-07-08T03:38:28Z ERROR Reconciler error {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "e3f4cb62-5f7b-4680-a80e-047bf1629a25", "error": "failed to create EnvoyFilter: could not CREATE EnvoyFilter openshift-ingress/e2e-api-a-e8b238-authn-ssl: envoyfilters.networking.istio.io \"e2e-api-a-e8b238-authn-ssl\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T03:38:28Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "e4d84096-1c69-4935-99cb-2c2cc376071a", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress"} 2026-07-08T03:38:28Z DEBUG events Failed to reconcile EnvoyFilter: failed to create EnvoyFilter: could not CREATE EnvoyFilter openshift-ingress/e2e-api-a-e8b238-authn-ssl: envoyfilters.networking.istio.io "e2e-api-a-e8b238-authn-ssl" already exists {"type": "Warning", "object": {"kind":"Gateway","namespace":"openshift-ingress","name":"e2e-api-a-e8b238","uid":"a08f7dcb-b431-4082-b2d1-72456875353a","apiVersion":"gateway.networking.k8s.io/v1","resourceVersion":"53761"}, "reason": "ReconcileError"} 2026-07-08T03:38:28Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "e4d84096-1c69-4935-99cb-2c2cc376071a", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238-authn-ssl"} 2026-07-08T03:38:28Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "e4d84096-1c69-4935-99cb-2c2cc376071a", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress"} 2026-07-08T03:38:28Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "72c37f4c-e2f9-4a28-b6bd-00a96e2c5d02", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress"} 2026-07-08T03:38:28Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "72c37f4c-e2f9-4a28-b6bd-00a96e2c5d02", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238-authn-ssl"} 2026-07-08T03:38:28Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "72c37f4c-e2f9-4a28-b6bd-00a96e2c5d02", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress"} 2026-07-08T03:38:28Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "e689f5d2-1444-4f9d-aa84-29908269119f", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress"} 2026-07-08T03:38:28Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "e689f5d2-1444-4f9d-aa84-29908269119f", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238-authn-ssl"} 2026-07-08T03:38:28Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "e689f5d2-1444-4f9d-aa84-29908269119f", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress"} 2026-07-08T03:38:28Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "43d30cd9-939a-4cfe-a022-5811dac6f60f", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress"} 2026-07-08T03:38:28Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "43d30cd9-939a-4cfe-a022-5811dac6f60f", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238-authn-ssl"} 2026-07-08T03:38:28Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "43d30cd9-939a-4cfe-a022-5811dac6f60f", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress"} 2026-07-08T03:38:28Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "bf1b56a5-e9bc-4d06-b2d7-4e8661154912", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress"} 2026-07-08T03:38:28Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "bf1b56a5-e9bc-4d06-b2d7-4e8661154912", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238-authn-ssl"} 2026-07-08T03:38:28Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "bf1b56a5-e9bc-4d06-b2d7-4e8661154912", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress"} 2026-07-08T03:38:29Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "2480a6d7-5c4f-4532-8fee-175a79984269", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress"} 2026-07-08T03:38:29Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "2480a6d7-5c4f-4532-8fee-175a79984269", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238-authn-ssl"} 2026-07-08T03:38:29Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "2480a6d7-5c4f-4532-8fee-175a79984269", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress"} 2026-07-08T03:38:31Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-e8b238"}, "namespace": "ai-tenant-e2e-api-a-e8b238", "name": "openshift-service-ca.crt", "reconcileID": "a044dd0c-6ce0-4eef-8d75-9a05d7d61583", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-e8b238"} 2026-07-08T03:38:31Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-e8b238"}, "namespace": "ai-tenant-e2e-api-a-e8b238", "name": "odh-trusted-ca-bundle", "reconcileID": "acece190-7c1e-488b-bfdb-609a3e9a997c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-e8b238"} 2026-07-08T03:38:31Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-e8b238"}, "namespace": "ai-tenant-e2e-api-a-e8b238", "name": "openshift-service-ca.crt", "reconcileID": "07be8076-eff8-4ab7-96d2-f117862c2085", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-e8b238", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T03:38:31Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-e8b238"}, "namespace": "ai-tenant-e2e-api-a-e8b238", "name": "openshift-service-ca.crt", "reconcileID": "07be8076-eff8-4ab7-96d2-f117862c2085", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-e8b238"} 2026-07-08T03:38:31Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-e8b238"}, "namespace": "ai-tenant-e2e-api-a-e8b238", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "4eaf39d9-22f9-4760-8604-996953826142", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-e8b238"} 2026-07-08T03:38:31Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-e8b238"}, "namespace": "ai-tenant-e2e-api-a-e8b238", "name": "openshift-service-ca.crt", "reconcileID": "5a1ad7f4-b57c-4d49-bce9-df0d2d08330c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-e8b238"} 2026-07-08T03:38:31Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-e8b238"}, "namespace": "ai-tenant-e2e-api-a-e8b238", "name": "odh-trusted-ca-bundle", "reconcileID": "fd2296b7-65aa-4c7a-9e37-ba22b0fc9a28", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-e8b238", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T03:38:31Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-e8b238"}, "namespace": "ai-tenant-e2e-api-a-e8b238", "name": "odh-trusted-ca-bundle", "reconcileID": "fd2296b7-65aa-4c7a-9e37-ba22b0fc9a28", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-e8b238"} 2026-07-08T03:38:31Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-e8b238"}, "namespace": "ai-tenant-e2e-api-a-e8b238", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d6b0ca2f-7856-4099-a7c8-f479f0d7cc49", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-e8b238"} 2026-07-08T03:38:31Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "0e878e93-2aa9-4866-8617-28fb913e74f3", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress"} 2026-07-08T03:38:31Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "0e878e93-2aa9-4866-8617-28fb913e74f3", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238-authn-ssl"} 2026-07-08T03:38:31Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "0e878e93-2aa9-4866-8617-28fb913e74f3", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress"} 2026-07-08T03:38:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "3ba44732-8572-4270-928c-4d660fb9ad51", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T03:38:36Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "a7cafe93-5874-4b45-8860-9b572b90a02d", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress"} 2026-07-08T03:38:36Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "a7cafe93-5874-4b45-8860-9b572b90a02d", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa-authn-ssl"} 2026-07-08T03:38:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "a7cafe93-5874-4b45-8860-9b572b90a02d", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress"} 2026-07-08T03:38:36Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "96dc8181-6cd3-4a67-817c-11d82306c7ab", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress"} 2026-07-08T03:38:36Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "96dc8181-6cd3-4a67-817c-11d82306c7ab", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa-authn-ssl"} 2026-07-08T03:38:36Z ERROR Reconciler error {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "96dc8181-6cd3-4a67-817c-11d82306c7ab", "error": "failed to create EnvoyFilter: could not CREATE EnvoyFilter openshift-ingress/e2e-api-b-da32fa-authn-ssl: envoyfilters.networking.istio.io \"e2e-api-b-da32fa-authn-ssl\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T03:38:36Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "ce3e21c3-4ab8-48f6-97cb-6328f15b595e", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress"} 2026-07-08T03:38:36Z DEBUG events Failed to reconcile EnvoyFilter: failed to create EnvoyFilter: could not CREATE EnvoyFilter openshift-ingress/e2e-api-b-da32fa-authn-ssl: envoyfilters.networking.istio.io "e2e-api-b-da32fa-authn-ssl" already exists {"type": "Warning", "object": {"kind":"Gateway","namespace":"openshift-ingress","name":"e2e-api-b-da32fa","uid":"6750f157-ecce-4f79-ad4d-da2a6529b60a","apiVersion":"gateway.networking.k8s.io/v1","resourceVersion":"54051"}, "reason": "ReconcileError"} 2026-07-08T03:38:36Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "ce3e21c3-4ab8-48f6-97cb-6328f15b595e", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa-authn-ssl"} 2026-07-08T03:38:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "ce3e21c3-4ab8-48f6-97cb-6328f15b595e", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress"} 2026-07-08T03:38:36Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "2bce2225-a4a3-4ef9-bd2d-bdc71b85bda7", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress"} 2026-07-08T03:38:36Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "2bce2225-a4a3-4ef9-bd2d-bdc71b85bda7", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa-authn-ssl"} 2026-07-08T03:38:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "2bce2225-a4a3-4ef9-bd2d-bdc71b85bda7", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress"} 2026-07-08T03:38:36Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "bf8c12fd-e442-46b7-bcf6-eea78a1f9d61", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress"} 2026-07-08T03:38:36Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "bf8c12fd-e442-46b7-bcf6-eea78a1f9d61", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa-authn-ssl"} 2026-07-08T03:38:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "bf8c12fd-e442-46b7-bcf6-eea78a1f9d61", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress"} 2026-07-08T03:38:36Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "afbb6a17-675a-4d21-a874-2252aeeda147", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress"} 2026-07-08T03:38:36Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "afbb6a17-675a-4d21-a874-2252aeeda147", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa-authn-ssl"} 2026-07-08T03:38:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "afbb6a17-675a-4d21-a874-2252aeeda147", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress"} 2026-07-08T03:38:36Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "cfbc1b2e-69a3-4f48-85fc-77cb744ed4f4", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress"} 2026-07-08T03:38:36Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "cfbc1b2e-69a3-4f48-85fc-77cb744ed4f4", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa-authn-ssl"} 2026-07-08T03:38:36Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "cfbc1b2e-69a3-4f48-85fc-77cb744ed4f4", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress"} 2026-07-08T03:38:37Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "ba29cbdb-90e8-4672-b587-291e765fc0d0", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress"} 2026-07-08T03:38:37Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "ba29cbdb-90e8-4672-b587-291e765fc0d0", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa-authn-ssl"} 2026-07-08T03:38:37Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "ba29cbdb-90e8-4672-b587-291e765fc0d0", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress"} 2026-07-08T03:38:39Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-b-da32fa"}, "namespace": "ai-tenant-e2e-api-b-da32fa", "name": "openshift-service-ca.crt", "reconcileID": "1a51462d-9a33-45a4-839a-64daa5c4974c", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-b-da32fa"} 2026-07-08T03:38:39Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-b-da32fa"}, "namespace": "ai-tenant-e2e-api-b-da32fa", "name": "odh-trusted-ca-bundle", "reconcileID": "a85e6b28-7e0a-4510-b41e-5484e2460cdb", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-b-da32fa"} 2026-07-08T03:38:39Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-b-da32fa"}, "namespace": "ai-tenant-e2e-api-b-da32fa", "name": "openshift-service-ca.crt", "reconcileID": "2f770455-60b0-4923-943b-5275431b56e0", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-b-da32fa", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T03:38:39Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-b-da32fa"}, "namespace": "ai-tenant-e2e-api-b-da32fa", "name": "openshift-service-ca.crt", "reconcileID": "2f770455-60b0-4923-943b-5275431b56e0", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-b-da32fa"} 2026-07-08T03:38:39Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-da32fa"}, "namespace": "ai-tenant-e2e-api-b-da32fa", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ad4b41eb-4fab-47cf-8c73-3abf3b1bcde2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-b-da32fa"} 2026-07-08T03:38:39Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-b-da32fa"}, "namespace": "ai-tenant-e2e-api-b-da32fa", "name": "openshift-service-ca.crt", "reconcileID": "b122c6c0-10b0-4772-8ec3-3a64982f6bd5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-b-da32fa"} 2026-07-08T03:38:39Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-b-da32fa"}, "namespace": "ai-tenant-e2e-api-b-da32fa", "name": "odh-trusted-ca-bundle", "reconcileID": "5e1c9d9e-b509-4b99-95aa-c9750824913c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-b-da32fa", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T03:38:39Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-b-da32fa"}, "namespace": "ai-tenant-e2e-api-b-da32fa", "name": "odh-trusted-ca-bundle", "reconcileID": "5e1c9d9e-b509-4b99-95aa-c9750824913c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-b-da32fa"} 2026-07-08T03:38:39Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-da32fa"}, "namespace": "ai-tenant-e2e-api-b-da32fa", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f2719761-c1ba-4156-8caa-0df45a1fd809", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-b-da32fa"} 2026-07-08T03:38:40Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "cbbc6efc-41fb-4018-a148-61e4b942c6fb", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress"} 2026-07-08T03:38:40Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "cbbc6efc-41fb-4018-a148-61e4b942c6fb", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa-authn-ssl"} 2026-07-08T03:38:40Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "cbbc6efc-41fb-4018-a148-61e4b942c6fb", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress"} 2026-07-08T03:38:40Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "b09440a5-e2d0-4eeb-bb58-b555c15e0e26", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T03:38:41Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "4007a28b-0f54-49b0-8d3a-48f7985b7c59", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress"} 2026-07-08T03:38:41Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "4007a28b-0f54-49b0-8d3a-48f7985b7c59", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238-authn-ssl"} 2026-07-08T03:38:41Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "4007a28b-0f54-49b0-8d3a-48f7985b7c59", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress"} 2026-07-08T03:38:50Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "13f8c620-85fa-4ee4-85a8-9cf3fd17076c", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress"} 2026-07-08T03:38:50Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "13f8c620-85fa-4ee4-85a8-9cf3fd17076c", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa-authn-ssl"} 2026-07-08T03:38:50Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "13f8c620-85fa-4ee4-85a8-9cf3fd17076c", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress"} 2026-07-08T03:38:51Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "6916f4ee-de43-4141-909c-a3688fef93b4", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T03:38:52Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "b5cd2efc-a85c-411b-b7a2-b9c8a0695bfc", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T03:38:53Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "417eb59c-9913-4697-89c3-d3cef9de3207", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress"} 2026-07-08T03:38:53Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "417eb59c-9913-4697-89c3-d3cef9de3207", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238-authn-ssl"} 2026-07-08T03:38:53Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "417eb59c-9913-4697-89c3-d3cef9de3207", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress"} 2026-07-08T03:38:58Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-e8b238"}, "namespace": "ai-tenant-e2e-api-a-e8b238", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "75309236-d530-4745-a5fe-338c025c126c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-e8b238"} 2026-07-08T03:38:58Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-e8b238"}, "namespace": "ai-tenant-e2e-api-a-e8b238", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "75309236-d530-4745-a5fe-338c025c126c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-e8b238", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T03:38:58Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-e8b238"}, "namespace": "ai-tenant-e2e-api-a-e8b238", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "75309236-d530-4745-a5fe-338c025c126c", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-api-a-e8b238 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T03:38:58Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-e8b238"}, "namespace": "ai-tenant-e2e-api-a-e8b238", "name": "odh-trusted-ca-bundle", "reconcileID": "9de0f15f-eb52-4d95-93a7-8230c5602648", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-e8b238"} 2026-07-08T03:38:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-a-e8b238"}, "namespace": "ai-tenant-e2e-api-a-e8b238", "name": "odh-trusted-ca-bundle", "reconcileID": "9de0f15f-eb52-4d95-93a7-8230c5602648", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-a-e8b238"} 2026-07-08T03:38:58Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-e8b238"}, "namespace": "ai-tenant-e2e-api-a-e8b238", "name": "openshift-service-ca.crt", "reconcileID": "9ad32cb5-b568-4310-b74f-285fbda1df46", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-e8b238"} 2026-07-08T03:38:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-a-e8b238"}, "namespace": "ai-tenant-e2e-api-a-e8b238", "name": "openshift-service-ca.crt", "reconcileID": "9ad32cb5-b568-4310-b74f-285fbda1df46", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-a-e8b238"} 2026-07-08T03:38:58Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-e8b238"}, "namespace": "ai-tenant-e2e-api-a-e8b238", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7fa9362e-9a97-4dcd-bdea-ced71cb5be33", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-e8b238"} 2026-07-08T03:38:58Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-a-e8b238"}, "namespace": "ai-tenant-e2e-api-a-e8b238", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7fa9362e-9a97-4dcd-bdea-ced71cb5be33", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-a-e8b238"} 2026-07-08T03:38:59Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-a-e8b238","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-a-e8b238", "reconcileID": "c6c2b271-b373-4773-8d29-16c7b34187ca", "Gateway": "e2e-api-a-e8b238", "namespace": "openshift-ingress"} 2026-07-08T03:38:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "cdaf8308-b2f4-47ea-8382-03b1695b2355", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T03:39:00Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "6fa9f185-399b-4084-9a24-7ee6b016869d", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress"} 2026-07-08T03:39:00Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "6fa9f185-399b-4084-9a24-7ee6b016869d", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa-authn-ssl"} 2026-07-08T03:39:00Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "6fa9f185-399b-4084-9a24-7ee6b016869d", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress"} 2026-07-08T03:39:05Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-da32fa"}, "namespace": "ai-tenant-e2e-api-b-da32fa", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "27415024-c839-4f85-9337-42c0e28a8c86", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-b-da32fa"} 2026-07-08T03:39:05Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-da32fa"}, "namespace": "ai-tenant-e2e-api-b-da32fa", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "27415024-c839-4f85-9337-42c0e28a8c86", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-b-da32fa", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T03:39:05Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-da32fa"}, "namespace": "ai-tenant-e2e-api-b-da32fa", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "27415024-c839-4f85-9337-42c0e28a8c86", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-api-b-da32fa because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T03:39:05Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-b-da32fa"}, "namespace": "ai-tenant-e2e-api-b-da32fa", "name": "odh-trusted-ca-bundle", "reconcileID": "a4d1ecba-5cf2-4200-b821-70c10df1bb5e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-b-da32fa"} 2026-07-08T03:39:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-api-b-da32fa"}, "namespace": "ai-tenant-e2e-api-b-da32fa", "name": "odh-trusted-ca-bundle", "reconcileID": "a4d1ecba-5cf2-4200-b821-70c10df1bb5e", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-api-b-da32fa"} 2026-07-08T03:39:05Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-b-da32fa"}, "namespace": "ai-tenant-e2e-api-b-da32fa", "name": "openshift-service-ca.crt", "reconcileID": "c2f4513b-717c-400b-8ba4-ba19b08b60b9", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-b-da32fa"} 2026-07-08T03:39:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-api-b-da32fa"}, "namespace": "ai-tenant-e2e-api-b-da32fa", "name": "openshift-service-ca.crt", "reconcileID": "c2f4513b-717c-400b-8ba4-ba19b08b60b9", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-api-b-da32fa"} 2026-07-08T03:39:05Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-da32fa"}, "namespace": "ai-tenant-e2e-api-b-da32fa", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2903c971-1359-4bb8-8f11-ab6d07600c96", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-b-da32fa"} 2026-07-08T03:39:05Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-api-b-da32fa"}, "namespace": "ai-tenant-e2e-api-b-da32fa", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2903c971-1359-4bb8-8f11-ab6d07600c96", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-api-b-da32fa"} 2026-07-08T03:39:06Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-api-b-da32fa","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-api-b-da32fa", "reconcileID": "ba35864b-34d6-4542-b581-4d20bf0669be", "Gateway": "e2e-api-b-da32fa", "namespace": "openshift-ingress"} 2026-07-08T03:39:06Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "2c1eaff8-a936-4c15-8676-829a3172fd4e", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress"} 2026-07-08T03:39:06Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "2c1eaff8-a936-4c15-8676-829a3172fd4e", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a-authn-ssl"} 2026-07-08T03:39:06Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "2c1eaff8-a936-4c15-8676-829a3172fd4e", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress"} 2026-07-08T03:39:06Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "1cdb6393-df1e-4f20-8035-2d43cda09782", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress"} 2026-07-08T03:39:06Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "1cdb6393-df1e-4f20-8035-2d43cda09782", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a-authn-ssl"} 2026-07-08T03:39:06Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "1cdb6393-df1e-4f20-8035-2d43cda09782", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress"} 2026-07-08T03:39:06Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "45952898-a5c7-4c08-9e28-f80284beace2", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress"} 2026-07-08T03:39:06Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "45952898-a5c7-4c08-9e28-f80284beace2", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a-authn-ssl"} 2026-07-08T03:39:06Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "45952898-a5c7-4c08-9e28-f80284beace2", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress"} 2026-07-08T03:39:06Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "c978a9c7-2c4a-46f3-b1df-63d417c2bc38", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress"} 2026-07-08T03:39:06Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "c978a9c7-2c4a-46f3-b1df-63d417c2bc38", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a-authn-ssl"} 2026-07-08T03:39:07Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "c978a9c7-2c4a-46f3-b1df-63d417c2bc38", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress"} 2026-07-08T03:39:07Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "5cfa7e5a-f1a2-42eb-9094-d4e6e0c789c4", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress"} 2026-07-08T03:39:07Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "5cfa7e5a-f1a2-42eb-9094-d4e6e0c789c4", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a-authn-ssl"} 2026-07-08T03:39:07Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "5cfa7e5a-f1a2-42eb-9094-d4e6e0c789c4", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress"} 2026-07-08T03:39:07Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "52f61644-11ab-409e-bfca-5150c619cc81", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress"} 2026-07-08T03:39:07Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "52f61644-11ab-409e-bfca-5150c619cc81", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a-authn-ssl"} 2026-07-08T03:39:07Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "52f61644-11ab-409e-bfca-5150c619cc81", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress"} 2026-07-08T03:39:07Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "c9ed4e3f-0f5b-4d8c-874a-de5b7b923a20", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress"} 2026-07-08T03:39:07Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "c9ed4e3f-0f5b-4d8c-874a-de5b7b923a20", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a-authn-ssl"} 2026-07-08T03:39:07Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "c9ed4e3f-0f5b-4d8c-874a-de5b7b923a20", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress"} 2026-07-08T03:39:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-788e6a"}, "namespace": "ai-tenant-e2e-shared-a-788e6a", "name": "openshift-service-ca.crt", "reconcileID": "db1ca25f-0079-48f2-af6b-38c8345ccbfa", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-a-788e6a"} 2026-07-08T03:39:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-a-788e6a"}, "namespace": "ai-tenant-e2e-shared-a-788e6a", "name": "odh-trusted-ca-bundle", "reconcileID": "6c03218c-cea5-4dc7-8436-ad6eec1f6009", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-788e6a"} 2026-07-08T03:39:09Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-788e6a"}, "namespace": "ai-tenant-e2e-shared-a-788e6a", "name": "openshift-service-ca.crt", "reconcileID": "1de3def6-68ff-4745-8b8a-f6b6574ea6e8", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-a-788e6a", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T03:39:09Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-788e6a"}, "namespace": "ai-tenant-e2e-shared-a-788e6a", "name": "openshift-service-ca.crt", "reconcileID": "1de3def6-68ff-4745-8b8a-f6b6574ea6e8", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-a-788e6a"} 2026-07-08T03:39:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-788e6a"}, "namespace": "ai-tenant-e2e-shared-a-788e6a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "d59a3877-6419-402e-8622-2a33e0b0a65e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-788e6a"} 2026-07-08T03:39:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-788e6a"}, "namespace": "ai-tenant-e2e-shared-a-788e6a", "name": "openshift-service-ca.crt", "reconcileID": "b74b6328-67c1-496f-9f0a-a981e1e59724", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-a-788e6a"} 2026-07-08T03:39:09Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-a-788e6a"}, "namespace": "ai-tenant-e2e-shared-a-788e6a", "name": "odh-trusted-ca-bundle", "reconcileID": "439d0a21-6605-40ea-b551-8d6352dd5d4f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-788e6a", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T03:39:09Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-a-788e6a"}, "namespace": "ai-tenant-e2e-shared-a-788e6a", "name": "odh-trusted-ca-bundle", "reconcileID": "439d0a21-6605-40ea-b551-8d6352dd5d4f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-788e6a"} 2026-07-08T03:39:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-788e6a"}, "namespace": "ai-tenant-e2e-shared-a-788e6a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ef317a05-9955-4d1b-b718-a8a4acb3805b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-788e6a"} 2026-07-08T03:39:10Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "55c7d658-57e0-44ad-a8f8-e8b1351be70f", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress"} 2026-07-08T03:39:10Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "55c7d658-57e0-44ad-a8f8-e8b1351be70f", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a-authn-ssl"} 2026-07-08T03:39:10Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "55c7d658-57e0-44ad-a8f8-e8b1351be70f", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress"} 2026-07-08T03:39:10Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "03f715a7-c218-4e78-afbb-b07ae6d644de", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T03:39:14Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "28e4447a-c605-42f9-9a91-7dd177f3b668", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress"} 2026-07-08T03:39:14Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "28e4447a-c605-42f9-9a91-7dd177f3b668", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress", "name": "e2e-shared-b-705492-authn-ssl"} 2026-07-08T03:39:14Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "28e4447a-c605-42f9-9a91-7dd177f3b668", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress"} 2026-07-08T03:39:14Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "8a14f8e6-f94d-431a-b81c-86b6bca042b2", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress"} 2026-07-08T03:39:14Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "8a14f8e6-f94d-431a-b81c-86b6bca042b2", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress", "name": "e2e-shared-b-705492-authn-ssl"} 2026-07-08T03:39:14Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "8a14f8e6-f94d-431a-b81c-86b6bca042b2", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress"} 2026-07-08T03:39:14Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "7856c736-3452-4426-8f55-934bada7275e", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress"} 2026-07-08T03:39:14Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "7856c736-3452-4426-8f55-934bada7275e", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress", "name": "e2e-shared-b-705492-authn-ssl"} 2026-07-08T03:39:14Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "7856c736-3452-4426-8f55-934bada7275e", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress"} 2026-07-08T03:39:14Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "59170735-6fac-4743-89c0-87b99a3865b8", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress"} 2026-07-08T03:39:14Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "59170735-6fac-4743-89c0-87b99a3865b8", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress", "name": "e2e-shared-b-705492-authn-ssl"} 2026-07-08T03:39:14Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "59170735-6fac-4743-89c0-87b99a3865b8", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress"} 2026-07-08T03:39:14Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "54d74ad1-d328-4736-a16c-a0b06b7c4a73", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress"} 2026-07-08T03:39:14Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "54d74ad1-d328-4736-a16c-a0b06b7c4a73", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress", "name": "e2e-shared-b-705492-authn-ssl"} 2026-07-08T03:39:14Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "54d74ad1-d328-4736-a16c-a0b06b7c4a73", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress"} 2026-07-08T03:39:14Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "802962d7-280f-4c16-a241-f8675263bc08", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress"} 2026-07-08T03:39:14Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "802962d7-280f-4c16-a241-f8675263bc08", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress", "name": "e2e-shared-b-705492-authn-ssl"} 2026-07-08T03:39:14Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "802962d7-280f-4c16-a241-f8675263bc08", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress"} 2026-07-08T03:39:14Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "aa2f5926-f421-48e5-bb39-14c117b71146", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress"} 2026-07-08T03:39:14Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "aa2f5926-f421-48e5-bb39-14c117b71146", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress", "name": "e2e-shared-b-705492-authn-ssl"} 2026-07-08T03:39:14Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "aa2f5926-f421-48e5-bb39-14c117b71146", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress"} 2026-07-08T03:39:15Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "9e0a394c-210a-4df4-90d5-fd871b1f800e", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress"} 2026-07-08T03:39:15Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "9e0a394c-210a-4df4-90d5-fd871b1f800e", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a-authn-ssl"} 2026-07-08T03:39:15Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "9e0a394c-210a-4df4-90d5-fd871b1f800e", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress"} 2026-07-08T03:39:16Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "895f6c08-2f6c-4f9a-b0f6-7be2131fc782", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress"} 2026-07-08T03:39:16Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "895f6c08-2f6c-4f9a-b0f6-7be2131fc782", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress", "name": "e2e-shared-b-705492-authn-ssl"} 2026-07-08T03:39:16Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "895f6c08-2f6c-4f9a-b0f6-7be2131fc782", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress"} 2026-07-08T03:39:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-b-705492"}, "namespace": "ai-tenant-e2e-shared-b-705492", "name": "openshift-service-ca.crt", "reconcileID": "8555d7c0-e362-460a-9ea4-088a93374862", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-b-705492"} 2026-07-08T03:39:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-b-705492"}, "namespace": "ai-tenant-e2e-shared-b-705492", "name": "odh-trusted-ca-bundle", "reconcileID": "d9214ffc-b00c-41d3-a94e-121e05b94377", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-705492"} 2026-07-08T03:39:17Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-b-705492"}, "namespace": "ai-tenant-e2e-shared-b-705492", "name": "openshift-service-ca.crt", "reconcileID": "cff7a908-587f-4d2a-9b67-b7cf2472eb88", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-b-705492", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T03:39:17Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-b-705492"}, "namespace": "ai-tenant-e2e-shared-b-705492", "name": "openshift-service-ca.crt", "reconcileID": "cff7a908-587f-4d2a-9b67-b7cf2472eb88", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-b-705492"} 2026-07-08T03:39:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-705492"}, "namespace": "ai-tenant-e2e-shared-b-705492", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5e49bb34-03f9-42e7-972e-bcab78188d7e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-705492"} 2026-07-08T03:39:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-b-705492"}, "namespace": "ai-tenant-e2e-shared-b-705492", "name": "openshift-service-ca.crt", "reconcileID": "71d7b25e-3a39-4621-85b9-070668762422", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-b-705492"} 2026-07-08T03:39:18Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-b-705492"}, "namespace": "ai-tenant-e2e-shared-b-705492", "name": "odh-trusted-ca-bundle", "reconcileID": "5fa497da-994e-454d-a1f5-a9fab65d1bd1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-705492", "update": "odh-kserve-custom-ca-bundle"} 2026-07-08T03:39:18Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-b-705492"}, "namespace": "ai-tenant-e2e-shared-b-705492", "name": "odh-trusted-ca-bundle", "reconcileID": "5fa497da-994e-454d-a1f5-a9fab65d1bd1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-705492"} 2026-07-08T03:39:18Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-705492"}, "namespace": "ai-tenant-e2e-shared-b-705492", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6ae68306-fed4-4d7f-b438-611666e9fc0f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-705492"} 2026-07-08T03:39:18Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "9e9a853c-720c-4985-8ff2-4d27ffb5914e", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress"} 2026-07-08T03:39:18Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "9e9a853c-720c-4985-8ff2-4d27ffb5914e", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress", "name": "e2e-shared-b-705492-authn-ssl"} 2026-07-08T03:39:18Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "9e9a853c-720c-4985-8ff2-4d27ffb5914e", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress"} 2026-07-08T03:39:18Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "e1810a29-928e-4dc8-ac5e-189b6fb7a0d2", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T03:39:23Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "74f991f9-fa30-42c8-8c02-bffd3a616b15", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress"} 2026-07-08T03:39:23Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "74f991f9-fa30-42c8-8c02-bffd3a616b15", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress", "name": "e2e-shared-b-705492-authn-ssl"} 2026-07-08T03:39:23Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "74f991f9-fa30-42c8-8c02-bffd3a616b15", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress"} 2026-07-08T03:39:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "270b4350-3118-433a-bc40-b2420d2e6787", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T03:39:39Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "277f24af-1533-450d-bb67-b67580ce178f", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T03:42:27Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "0eddc516-22fc-4b02-a073-2ada642d8c6f", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T03:42:28Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "c455ecc8-dd2a-4f81-a062-d562a140049d", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress"} 2026-07-08T03:42:28Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "c455ecc8-dd2a-4f81-a062-d562a140049d", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a-authn-ssl"} 2026-07-08T03:42:28Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "c455ecc8-dd2a-4f81-a062-d562a140049d", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress"} 2026-07-08T03:42:32Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-788e6a"}, "namespace": "ai-tenant-e2e-shared-a-788e6a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2a04da0d-cee2-465a-a427-9e145466b5b9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-788e6a"} 2026-07-08T03:42:32Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-788e6a"}, "namespace": "ai-tenant-e2e-shared-a-788e6a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2a04da0d-cee2-465a-a427-9e145466b5b9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-788e6a", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T03:42:32Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-788e6a"}, "namespace": "ai-tenant-e2e-shared-a-788e6a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2a04da0d-cee2-465a-a427-9e145466b5b9", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-788e6a because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T03:42:32Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-a-788e6a"}, "namespace": "ai-tenant-e2e-shared-a-788e6a", "name": "odh-trusted-ca-bundle", "reconcileID": "736e268b-18f4-4d7a-97d9-5ada4e9d33a8", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-788e6a"} 2026-07-08T03:42:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-a-788e6a"}, "namespace": "ai-tenant-e2e-shared-a-788e6a", "name": "odh-trusted-ca-bundle", "reconcileID": "736e268b-18f4-4d7a-97d9-5ada4e9d33a8", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-788e6a"} 2026-07-08T03:42:32Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-788e6a"}, "namespace": "ai-tenant-e2e-shared-a-788e6a", "name": "openshift-service-ca.crt", "reconcileID": "99727ea1-bc10-4f8e-8c59-885011f5bdf5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-a-788e6a"} 2026-07-08T03:42:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-a-788e6a"}, "namespace": "ai-tenant-e2e-shared-a-788e6a", "name": "openshift-service-ca.crt", "reconcileID": "99727ea1-bc10-4f8e-8c59-885011f5bdf5", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-a-788e6a"} 2026-07-08T03:42:32Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-788e6a"}, "namespace": "ai-tenant-e2e-shared-a-788e6a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "11f464fa-a8c0-46b9-88ce-d8a3374e7b8d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-788e6a"} 2026-07-08T03:42:32Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-a-788e6a"}, "namespace": "ai-tenant-e2e-shared-a-788e6a", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "11f464fa-a8c0-46b9-88ce-d8a3374e7b8d", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-a-788e6a"} 2026-07-08T03:42:33Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-a-788e6a","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-a-788e6a", "reconcileID": "fb18562a-4033-4f49-b86c-2a06451fe173", "Gateway": "e2e-shared-a-788e6a", "namespace": "openshift-ingress"} 2026-07-08T03:42:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"odh-ai-gateway-infra"}, "namespace": "odh-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "ab2c7b7e-f064-4903-84fc-cfdfe5397969", "ConfigMap": "openshift-service-ca.crt", "namespace": "odh-ai-gateway-infra"} 2026-07-08T03:42:34Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "2fc868a1-eddc-4bd2-8836-c2bf193f89ce", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress"} 2026-07-08T03:42:34Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "2fc868a1-eddc-4bd2-8836-c2bf193f89ce", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress", "name": "e2e-shared-b-705492-authn-ssl"} 2026-07-08T03:42:34Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "2fc868a1-eddc-4bd2-8836-c2bf193f89ce", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress"} 2026-07-08T03:42:39Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-705492"}, "namespace": "ai-tenant-e2e-shared-b-705492", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2b6c962b-34d8-4f6e-a296-2307e81288c9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-705492"} 2026-07-08T03:42:39Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-705492"}, "namespace": "ai-tenant-e2e-shared-b-705492", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2b6c962b-34d8-4f6e-a296-2307e81288c9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-705492", "create": "odh-kserve-custom-ca-bundle"} 2026-07-08T03:42:39Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-705492"}, "namespace": "ai-tenant-e2e-shared-b-705492", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2b6c962b-34d8-4f6e-a296-2307e81288c9", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-705492 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-08T03:42:39Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-b-705492"}, "namespace": "ai-tenant-e2e-shared-b-705492", "name": "odh-trusted-ca-bundle", "reconcileID": "69add3f7-6e5e-4103-ae6c-8b90a38e6711", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-705492"} 2026-07-08T03:42:39Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"ai-tenant-e2e-shared-b-705492"}, "namespace": "ai-tenant-e2e-shared-b-705492", "name": "odh-trusted-ca-bundle", "reconcileID": "69add3f7-6e5e-4103-ae6c-8b90a38e6711", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-705492"} 2026-07-08T03:42:39Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-b-705492"}, "namespace": "ai-tenant-e2e-shared-b-705492", "name": "openshift-service-ca.crt", "reconcileID": "14815b4d-77c3-4dcf-95be-4911516e98ba", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-b-705492"} 2026-07-08T03:42:39Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"ai-tenant-e2e-shared-b-705492"}, "namespace": "ai-tenant-e2e-shared-b-705492", "name": "openshift-service-ca.crt", "reconcileID": "14815b4d-77c3-4dcf-95be-4911516e98ba", "ConfigMap": "openshift-service-ca.crt", "namespace": "ai-tenant-e2e-shared-b-705492"} 2026-07-08T03:42:39Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-705492"}, "namespace": "ai-tenant-e2e-shared-b-705492", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8b32f473-d0ac-477e-9fca-b9c436738efb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-705492"} 2026-07-08T03:42:39Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"ai-tenant-e2e-shared-b-705492"}, "namespace": "ai-tenant-e2e-shared-b-705492", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8b32f473-d0ac-477e-9fca-b9c436738efb", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "ai-tenant-e2e-shared-b-705492"} 2026-07-08T03:42:40Z DEBUG Gateway not found, resources owned by Gateway will be garbage collected {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-shared-b-705492","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-shared-b-705492", "reconcileID": "ad3026b1-41e3-425a-9927-04c056c68e8e", "Gateway": "e2e-shared-b-705492", "namespace": "openshift-ingress"}