2026-06-05T14:33:32Z INFO Starting workers {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "worker count": 1} 2026-06-05T14:33:32Z INFO Starting workers {"controller": "inferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "InferenceService", "worker count": 1} 2026-06-05T14:33:33Z ERROR fail to get ray ca cert secret {"ResourceName": "openshift-monitoring", "Namespace": "openshift-monitoring", "secret": "ray-ca-tls", "namespace": "opendatahub", "error": "Secret \"ray-ca-tls\" not found"} github.com/opendatahub-io/odh-model-controller/internal/controller/serving.(*ServingRuntimeReconciler).reconcileMultiNodeSR /workspace/internal/controller/serving/servingruntime_controller.go:289 github.com/opendatahub-io/odh-model-controller/internal/controller/serving.(*ServingRuntimeReconciler).Reconcile /workspace/internal/controller/serving/servingruntime_controller.go:270 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-05T14:33:33Z ERROR Reconciler error {"controller": "servingruntime", "controllerGroup": "serving.kserve.io", "controllerKind": "ServingRuntime", "ServingRuntime": {"name":"openshift-monitoring","namespace":"openshift-monitoring"}, "namespace": "openshift-monitoring", "name": "openshift-monitoring", "reconcileID": "fac91d44-4c19-4a1e-bcb6-ecb16202b801", "error": "Secret \"ray-ca-tls\" not found"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-05T14:33:33Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-05T14:33:33Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-05T14:33:36Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-05T14:33:36Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-05T14:33:36Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "openshift-monitoring", "Namespace": "openshift-monitoring"} 2026-06-05T14:33:36Z INFO Multi Node reconciling. {"ResourceName": "openshift-monitoring", "Namespace": "openshift-monitoring"} 2026-06-05T14:33:36Z INFO Multi Node reconciled successfully. {"ResourceName": "openshift-monitoring", "Namespace": "openshift-monitoring"} 2026-06-05T14:34:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"kuadrant-system"}, "namespace": "kuadrant-system", "name": "openshift-service-ca.crt", "reconcileID": "7c2abf32-cdd0-4bfa-8d88-56ac75f25b30", "ConfigMap": "openshift-service-ca.crt", "namespace": "kuadrant-system"} 2026-06-05T14:34:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"kuadrant-system"}, "namespace": "kuadrant-system", "name": "odh-trusted-ca-bundle", "reconcileID": "998f62c9-0ab2-48ed-b5c4-dd4a385c2986", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "kuadrant-system"} 2026-06-05T14:34:17Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"kuadrant-system"}, "namespace": "kuadrant-system", "name": "openshift-service-ca.crt", "reconcileID": "0e07eabf-7468-4a6d-958c-0144e8739a65", "ConfigMap": "openshift-service-ca.crt", "namespace": "kuadrant-system", "create": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:34:17Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"kuadrant-system"}, "namespace": "kuadrant-system", "name": "openshift-service-ca.crt", "reconcileID": "0e07eabf-7468-4a6d-958c-0144e8739a65", "ConfigMap": "openshift-service-ca.crt", "namespace": "kuadrant-system"} 2026-06-05T14:34:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"kuadrant-system"}, "namespace": "kuadrant-system", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ef971ab9-373e-42b3-b0d4-29b7ff4cb960", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "kuadrant-system"} 2026-06-05T14:34:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"kuadrant-system"}, "namespace": "kuadrant-system", "name": "openshift-service-ca.crt", "reconcileID": "f64ce3fc-d901-4fc3-ba75-b924966e48b4", "ConfigMap": "openshift-service-ca.crt", "namespace": "kuadrant-system"} 2026-06-05T14:34:17Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"kuadrant-system"}, "namespace": "kuadrant-system", "name": "odh-trusted-ca-bundle", "reconcileID": "3bfa3283-4c86-4c56-8122-cb579288aa66", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "kuadrant-system", "update": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:34:17Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"kuadrant-system"}, "namespace": "kuadrant-system", "name": "odh-trusted-ca-bundle", "reconcileID": "3bfa3283-4c86-4c56-8122-cb579288aa66", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "kuadrant-system"} 2026-06-05T14:34:17Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"kuadrant-system"}, "namespace": "kuadrant-system", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a753a0dd-57a0-47c0-bbb6-fcb204e23fff", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "kuadrant-system"} 2026-06-05T14:35:41Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "cd5fd0fd-019b-49a9-a3b3-259e65cc0866", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:35:41Z INFO Creating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "cd5fd0fd-019b-49a9-a3b3-259e65cc0866", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:35:41Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "cd5fd0fd-019b-49a9-a3b3-259e65cc0866", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:35:41Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "c9b7f7ed-afea-4fc4-8ff4-a887b797be19", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:35:41Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "c9b7f7ed-afea-4fc4-8ff4-a887b797be19", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:35:41Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "c9b7f7ed-afea-4fc4-8ff4-a887b797be19", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:35:43Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "cd1658e6-e117-4c10-bf0b-ce5696d765fd", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:35:43Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "cd1658e6-e117-4c10-bf0b-ce5696d765fd", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:35:43Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "cd1658e6-e117-4c10-bf0b-ce5696d765fd", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:35:45Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "f0f8f35b-3fe2-44a8-a374-4d2a945a39f8", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:35:45Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "f0f8f35b-3fe2-44a8-a374-4d2a945a39f8", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:35:45Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "f0f8f35b-3fe2-44a8-a374-4d2a945a39f8", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:36:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"keycloak-system"}, "namespace": "keycloak-system", "name": "openshift-service-ca.crt", "reconcileID": "41be19c0-b067-4cee-8654-5d0551fb551e", "ConfigMap": "openshift-service-ca.crt", "namespace": "keycloak-system"} 2026-06-05T14:36:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"keycloak-system"}, "namespace": "keycloak-system", "name": "odh-trusted-ca-bundle", "reconcileID": "3865e42c-a937-4f3b-95f3-b07d4ef8a5d7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "keycloak-system"} 2026-06-05T14:36:28Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"keycloak-system"}, "namespace": "keycloak-system", "name": "openshift-service-ca.crt", "reconcileID": "d103662d-3628-4bab-a3bf-7697e9497e55", "ConfigMap": "openshift-service-ca.crt", "namespace": "keycloak-system", "create": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:36:28Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"keycloak-system"}, "namespace": "keycloak-system", "name": "openshift-service-ca.crt", "reconcileID": "d103662d-3628-4bab-a3bf-7697e9497e55", "ConfigMap": "openshift-service-ca.crt", "namespace": "keycloak-system"} 2026-06-05T14:36:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"keycloak-system"}, "namespace": "keycloak-system", "name": "openshift-service-ca.crt", "reconcileID": "098d7f76-8b28-42cc-ba7e-e20a8843cb7a", "ConfigMap": "openshift-service-ca.crt", "namespace": "keycloak-system"} 2026-06-05T14:36:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"keycloak-system"}, "namespace": "keycloak-system", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e3778113-a15e-4818-b1a3-29f04f5cd5a8", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "keycloak-system"} 2026-06-05T14:36:28Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"keycloak-system"}, "namespace": "keycloak-system", "name": "odh-trusted-ca-bundle", "reconcileID": "120b54d8-1ccf-4379-9f35-ac6372a7da28", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "keycloak-system", "update": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:36:28Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"keycloak-system"}, "namespace": "keycloak-system", "name": "odh-trusted-ca-bundle", "reconcileID": "120b54d8-1ccf-4379-9f35-ac6372a7da28", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "keycloak-system"} 2026-06-05T14:36:28Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"keycloak-system"}, "namespace": "keycloak-system", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "03a3b390-9087-4a0e-ab46-0f1bfd70ce0b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "keycloak-system"} 2026-06-05T14:37:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"models-as-a-service"}, "namespace": "models-as-a-service", "name": "openshift-service-ca.crt", "reconcileID": "8b1b0902-2c3a-4cc1-8caa-86f1587f2791", "ConfigMap": "openshift-service-ca.crt", "namespace": "models-as-a-service"} 2026-06-05T14:37:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"models-as-a-service"}, "namespace": "models-as-a-service", "name": "odh-trusted-ca-bundle", "reconcileID": "e13a94b1-41ae-4641-87c9-9a284a674d86", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "models-as-a-service"} 2026-06-05T14:37:38Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"models-as-a-service"}, "namespace": "models-as-a-service", "name": "openshift-service-ca.crt", "reconcileID": "fd3c31e4-6c36-4404-af7b-27b875406c28", "ConfigMap": "openshift-service-ca.crt", "namespace": "models-as-a-service", "create": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:37:38Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"models-as-a-service"}, "namespace": "models-as-a-service", "name": "openshift-service-ca.crt", "reconcileID": "fd3c31e4-6c36-4404-af7b-27b875406c28", "ConfigMap": "openshift-service-ca.crt", "namespace": "models-as-a-service"} 2026-06-05T14:37:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"redhat-ai-gateway-infra"}, "namespace": "redhat-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "320817c9-7027-4d62-9d41-da280971fecf", "ConfigMap": "openshift-service-ca.crt", "namespace": "redhat-ai-gateway-infra"} 2026-06-05T14:37:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"models-as-a-service"}, "namespace": "models-as-a-service", "name": "openshift-service-ca.crt", "reconcileID": "b61bfdcf-729f-4178-8063-7ff882011a68", "ConfigMap": "openshift-service-ca.crt", "namespace": "models-as-a-service"} 2026-06-05T14:37:38Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"redhat-ai-gateway-infra"}, "namespace": "redhat-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "996c3ad7-7eec-4f7c-8c58-d8bdb4c17010", "ConfigMap": "openshift-service-ca.crt", "namespace": "redhat-ai-gateway-infra", "create": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:37:38Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"redhat-ai-gateway-infra"}, "namespace": "redhat-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "996c3ad7-7eec-4f7c-8c58-d8bdb4c17010", "ConfigMap": "openshift-service-ca.crt", "namespace": "redhat-ai-gateway-infra"} 2026-06-05T14:37:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"redhat-ai-gateway-infra"}, "namespace": "redhat-ai-gateway-infra", "name": "odh-trusted-ca-bundle", "reconcileID": "0c1b2c3e-a2ec-4990-b945-283ed826f7ed", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "redhat-ai-gateway-infra"} 2026-06-05T14:37:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"models-as-a-service"}, "namespace": "models-as-a-service", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1476e55a-2be5-4288-9de2-b19395a5ebd1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "models-as-a-service"} 2026-06-05T14:37:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"redhat-ai-gateway-infra"}, "namespace": "redhat-ai-gateway-infra", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ea18576c-d13f-4bc2-807f-ffef2d42e27b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "redhat-ai-gateway-infra"} 2026-06-05T14:37:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"redhat-ai-gateway-infra"}, "namespace": "redhat-ai-gateway-infra", "name": "openshift-service-ca.crt", "reconcileID": "2e2cded5-53a5-4aa4-a43e-29550241c535", "ConfigMap": "openshift-service-ca.crt", "namespace": "redhat-ai-gateway-infra"} 2026-06-05T14:37:38Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"models-as-a-service"}, "namespace": "models-as-a-service", "name": "odh-trusted-ca-bundle", "reconcileID": "c252e083-63b4-4615-8ee1-93afdd10d1c3", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "models-as-a-service", "update": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:37:38Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"models-as-a-service"}, "namespace": "models-as-a-service", "name": "odh-trusted-ca-bundle", "reconcileID": "c252e083-63b4-4615-8ee1-93afdd10d1c3", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "models-as-a-service"} 2026-06-05T14:37:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"models-as-a-service"}, "namespace": "models-as-a-service", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7822f9f9-e829-455b-ab45-969725f1b8e4", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "models-as-a-service"} 2026-06-05T14:37:38Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"redhat-ai-gateway-infra"}, "namespace": "redhat-ai-gateway-infra", "name": "odh-trusted-ca-bundle", "reconcileID": "8a31b081-3133-45e0-b702-f8e0b93fb25c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "redhat-ai-gateway-infra", "update": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:37:38Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"redhat-ai-gateway-infra"}, "namespace": "redhat-ai-gateway-infra", "name": "odh-trusted-ca-bundle", "reconcileID": "8a31b081-3133-45e0-b702-f8e0b93fb25c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "redhat-ai-gateway-infra"} 2026-06-05T14:37:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"redhat-ai-gateway-infra"}, "namespace": "redhat-ai-gateway-infra", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0a26d7e6-e70b-46fe-a472-9292bd7a5425", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "redhat-ai-gateway-infra"} 2026-06-05T14:38:10Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0a6631dd-5190-446a-9b03-9f4e25e1937c", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:38:10Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0a6631dd-5190-446a-9b03-9f4e25e1937c", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:38:10Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0a6631dd-5190-446a-9b03-9f4e25e1937c", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:38:10Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "d1e7012b-f864-410b-8310-976cadf88bf1", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:38:10Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "d1e7012b-f864-410b-8310-976cadf88bf1", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:38:10Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "d1e7012b-f864-410b-8310-976cadf88bf1", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:39:01Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"llm"}, "namespace": "llm", "name": "openshift-service-ca.crt", "reconcileID": "99da1b73-ef00-4cbf-ae91-4459c2e34c00", "ConfigMap": "openshift-service-ca.crt", "namespace": "llm"} 2026-06-05T14:39:01Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"llm"}, "namespace": "llm", "name": "odh-trusted-ca-bundle", "reconcileID": "8a42de91-2f0c-4870-9ff8-cab52ac8c9b7", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "llm"} 2026-06-05T14:39:01Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"llm"}, "namespace": "llm", "name": "openshift-service-ca.crt", "reconcileID": "90c31946-74ba-4762-a767-84016c911d00", "ConfigMap": "openshift-service-ca.crt", "namespace": "llm", "create": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:39:01Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"llm"}, "namespace": "llm", "name": "openshift-service-ca.crt", "reconcileID": "90c31946-74ba-4762-a767-84016c911d00", "ConfigMap": "openshift-service-ca.crt", "namespace": "llm"} 2026-06-05T14:39:01Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"llm"}, "namespace": "llm", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "87c32ec2-5e53-4923-8f7a-f6f5bc6cbb7e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "llm"} 2026-06-05T14:39:01Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"llm"}, "namespace": "llm", "name": "openshift-service-ca.crt", "reconcileID": "7e2d0b57-fbc7-4dbc-a881-b659e368faff", "ConfigMap": "openshift-service-ca.crt", "namespace": "llm"} 2026-06-05T14:39:01Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"llm"}, "namespace": "llm", "name": "odh-trusted-ca-bundle", "reconcileID": "70e2a202-c03b-4863-9615-043af635adc1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "llm", "update": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:39:01Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"llm"}, "namespace": "llm", "name": "odh-trusted-ca-bundle", "reconcileID": "70e2a202-c03b-4863-9615-043af635adc1", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "llm"} 2026-06-05T14:39:01Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"llm"}, "namespace": "llm", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "423f6720-3d89-43f6-a8a6-70c8dce66e39", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "946036ed-5127-47cc-9498-8c03b9adfaf0", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "946036ed-5127-47cc-9498-8c03b9adfaf0", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "946036ed-5127-47cc-9498-8c03b9adfaf0", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "eae26f07-5727-4910-86a9-0a79b78e1c0f", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:39:02Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "eae26f07-5727-4910-86a9-0a79b78e1c0f", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:39:02Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "946036ed-5127-47cc-9498-8c03b9adfaf0", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm", "name": "e2e-distinct-2-simulated-kserve-route-authn"} 2026-06-05T14:39:02Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "946036ed-5127-47cc-9498-8c03b9adfaf0", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:39:02Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "946036ed-5127-47cc-9498-8c03b9adfaf0", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "eae26f07-5727-4910-86a9-0a79b78e1c0f", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:39:02Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "ade83018-b78a-406d-9c69-33bb5acd6ecb", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "ade83018-b78a-406d-9c69-33bb5acd6ecb", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "ade83018-b78a-406d-9c69-33bb5acd6ecb", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "a7b48ee5-89da-4df0-a3b9-51ab9a1d2521", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:39:02Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "ade83018-b78a-406d-9c69-33bb5acd6ecb", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm", "name": "e2e-distinct-simulated-kserve-route-authn"} 2026-06-05T14:39:02Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "ade83018-b78a-406d-9c69-33bb5acd6ecb", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:39:02Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "ade83018-b78a-406d-9c69-33bb5acd6ecb", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "a7b48ee5-89da-4df0-a3b9-51ab9a1d2521", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:39:02Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "a7b48ee5-89da-4df0-a3b9-51ab9a1d2521", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:39:02Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "ec31bf24-5e35-4133-89f0-425f2d023fe0", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "ec31bf24-5e35-4133-89f0-425f2d023fe0", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "ec31bf24-5e35-4133-89f0-425f2d023fe0", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "ecf5f356-937b-4802-a48c-5856adeb52c6", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:39:02Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "ec31bf24-5e35-4133-89f0-425f2d023fe0", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm", "name": "e2e-trlp-test-simulated-kserve-route-authn"} 2026-06-05T14:39:02Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "ec31bf24-5e35-4133-89f0-425f2d023fe0", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:39:02Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "ec31bf24-5e35-4133-89f0-425f2d023fe0", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "ecf5f356-937b-4802-a48c-5856adeb52c6", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:39:02Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "ecf5f356-937b-4802-a48c-5856adeb52c6", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:39:02Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "991a2665-a8ed-4428-913f-a8937aa81993", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "991a2665-a8ed-4428-913f-a8937aa81993", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "991a2665-a8ed-4428-913f-a8937aa81993", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "991a2665-a8ed-4428-913f-a8937aa81993", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm", "name": "e2e-distinct-2-simulated-kserve-route-authn"} 2026-06-05T14:39:02Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "991a2665-a8ed-4428-913f-a8937aa81993", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:39:02Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "991a2665-a8ed-4428-913f-a8937aa81993", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "ca72f5a8-889d-400f-b105-0ad2fca7abc7", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "ca72f5a8-889d-400f-b105-0ad2fca7abc7", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "ca72f5a8-889d-400f-b105-0ad2fca7abc7", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0dbf330d-ffe4-4b56-a115-eeac96b34ec5", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:39:02Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "ca72f5a8-889d-400f-b105-0ad2fca7abc7", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated-kserve-route-authn"} 2026-06-05T14:39:02Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "ca72f5a8-889d-400f-b105-0ad2fca7abc7", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:39:02Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "ca72f5a8-889d-400f-b105-0ad2fca7abc7", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0dbf330d-ffe4-4b56-a115-eeac96b34ec5", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:39:02Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0dbf330d-ffe4-4b56-a115-eeac96b34ec5", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:39:02Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "fb4353ce-91ee-42aa-8b7a-2a28b54a8a3f", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "fb4353ce-91ee-42aa-8b7a-2a28b54a8a3f", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "fb4353ce-91ee-42aa-8b7a-2a28b54a8a3f", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "b0e0f8d4-be64-4d7d-bddc-de917eb88def", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:39:02Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "fb4353ce-91ee-42aa-8b7a-2a28b54a8a3f", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm", "name": "facebook-opt-125m-simulated-kserve-route-authn"} 2026-06-05T14:39:02Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "fb4353ce-91ee-42aa-8b7a-2a28b54a8a3f", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:39:02Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "fb4353ce-91ee-42aa-8b7a-2a28b54a8a3f", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:39:02Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "b0e0f8d4-be64-4d7d-bddc-de917eb88def", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:39:02Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "b0e0f8d4-be64-4d7d-bddc-de917eb88def", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:39:02Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "513c0620-20cb-4035-868f-cebdaba75540", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "513c0620-20cb-4035-868f-cebdaba75540", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "513c0620-20cb-4035-868f-cebdaba75540", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-05T14:39:02Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "e4a49ef7-8aeb-48e0-8c60-bfbdb340cce8", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:39:02Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "513c0620-20cb-4035-868f-cebdaba75540", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm", "name": "premium-simulated-simulated-premium-kserve-route-authn"} 2026-06-05T14:39:02Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "513c0620-20cb-4035-868f-cebdaba75540", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:39:02Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "513c0620-20cb-4035-868f-cebdaba75540", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-05T14:39:02Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "e4a49ef7-8aeb-48e0-8c60-bfbdb340cce8", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:39:02Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "e4a49ef7-8aeb-48e0-8c60-bfbdb340cce8", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:39:34Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "39e726ac-3a28-48de-afab-08777c43b7de", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:39:34Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "39e726ac-3a28-48de-afab-08777c43b7de", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:39:34Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "39e726ac-3a28-48de-afab-08777c43b7de", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:39:35Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "f98da9e8-3418-4f2d-b762-41133874b532", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-05T14:39:35Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "f98da9e8-3418-4f2d-b762-41133874b532", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-05T14:39:35Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "f98da9e8-3418-4f2d-b762-41133874b532", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-05T14:39:35Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "f98da9e8-3418-4f2d-b762-41133874b532", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm", "name": "e2e-distinct-2-simulated-kserve-route-authn"} 2026-06-05T14:39:35Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "f98da9e8-3418-4f2d-b762-41133874b532", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:39:35Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "f98da9e8-3418-4f2d-b762-41133874b532", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-05T14:39:35Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "21f5397f-e70e-4dfe-b24b-a7844a96a8f9", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-05T14:39:35Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "21f5397f-e70e-4dfe-b24b-a7844a96a8f9", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-05T14:39:35Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "21f5397f-e70e-4dfe-b24b-a7844a96a8f9", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-05T14:39:35Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "21f5397f-e70e-4dfe-b24b-a7844a96a8f9", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm", "name": "e2e-distinct-simulated-kserve-route-authn"} 2026-06-05T14:39:35Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "21f5397f-e70e-4dfe-b24b-a7844a96a8f9", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:39:35Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "21f5397f-e70e-4dfe-b24b-a7844a96a8f9", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-05T14:39:39Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "7de1370d-e331-4785-8574-784b8a9b5c64", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:39:39Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "7de1370d-e331-4785-8574-784b8a9b5c64", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:39:39Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "7de1370d-e331-4785-8574-784b8a9b5c64", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:39:40Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "0824a240-d4e4-4665-a8b4-c149d18c71f1", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-05T14:39:40Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "0824a240-d4e4-4665-a8b4-c149d18c71f1", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-05T14:39:40Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "0824a240-d4e4-4665-a8b4-c149d18c71f1", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-05T14:39:40Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "0824a240-d4e4-4665-a8b4-c149d18c71f1", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm", "name": "e2e-distinct-simulated-kserve-route-authn"} 2026-06-05T14:39:40Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "0824a240-d4e4-4665-a8b4-c149d18c71f1", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:39:40Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "0824a240-d4e4-4665-a8b4-c149d18c71f1", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-05T14:39:40Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "08a071f8-1909-439a-a317-7d019278d7f3", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-05T14:39:40Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "08a071f8-1909-439a-a317-7d019278d7f3", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-05T14:39:40Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "08a071f8-1909-439a-a317-7d019278d7f3", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-05T14:39:40Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "08a071f8-1909-439a-a317-7d019278d7f3", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm", "name": "e2e-trlp-test-simulated-kserve-route-authn"} 2026-06-05T14:39:40Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "08a071f8-1909-439a-a317-7d019278d7f3", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:39:40Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "08a071f8-1909-439a-a317-7d019278d7f3", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-05T14:39:52Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "9116a67a-e351-4dee-b004-eed1f4658894", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:39:52Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "9116a67a-e351-4dee-b004-eed1f4658894", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:39:52Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "9116a67a-e351-4dee-b004-eed1f4658894", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:39:53Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "5a33ab03-8448-431c-944d-728ae9d17f6f", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-05T14:39:53Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "5a33ab03-8448-431c-944d-728ae9d17f6f", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-05T14:39:53Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "5a33ab03-8448-431c-944d-728ae9d17f6f", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-05T14:39:53Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "5a33ab03-8448-431c-944d-728ae9d17f6f", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm", "name": "e2e-trlp-test-simulated-kserve-route-authn"} 2026-06-05T14:39:53Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "5a33ab03-8448-431c-944d-728ae9d17f6f", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:39:53Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "5a33ab03-8448-431c-944d-728ae9d17f6f", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-05T14:39:53Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "723a0553-bec7-4a47-9cd3-2d74676a275c", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:39:53Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "723a0553-bec7-4a47-9cd3-2d74676a275c", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:39:53Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "723a0553-bec7-4a47-9cd3-2d74676a275c", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:39:53Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "723a0553-bec7-4a47-9cd3-2d74676a275c", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated-kserve-route-authn"} 2026-06-05T14:39:53Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "723a0553-bec7-4a47-9cd3-2d74676a275c", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:39:53Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "723a0553-bec7-4a47-9cd3-2d74676a275c", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:40:00Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "c43071de-bdca-4ef7-bd4c-3c14b1683441", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:40:00Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "c43071de-bdca-4ef7-bd4c-3c14b1683441", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:40:00Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "c43071de-bdca-4ef7-bd4c-3c14b1683441", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:40:01Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "db6301bc-3c57-46fb-b664-b9afd6d50335", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:40:01Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "db6301bc-3c57-46fb-b664-b9afd6d50335", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:40:01Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "db6301bc-3c57-46fb-b664-b9afd6d50335", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:40:01Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "db6301bc-3c57-46fb-b664-b9afd6d50335", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated-kserve-route-authn"} 2026-06-05T14:40:01Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "db6301bc-3c57-46fb-b664-b9afd6d50335", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:40:01Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "db6301bc-3c57-46fb-b664-b9afd6d50335", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:40:01Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "e53c44d3-085d-408a-a1cd-fccecb5ae40c", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:40:01Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "e53c44d3-085d-408a-a1cd-fccecb5ae40c", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:40:01Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "e53c44d3-085d-408a-a1cd-fccecb5ae40c", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:40:01Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "e53c44d3-085d-408a-a1cd-fccecb5ae40c", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm", "name": "facebook-opt-125m-simulated-kserve-route-authn"} 2026-06-05T14:40:01Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "e53c44d3-085d-408a-a1cd-fccecb5ae40c", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:40:01Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "e53c44d3-085d-408a-a1cd-fccecb5ae40c", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:40:18Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "7cba9a3e-4a6f-4d40-8cd2-913e2e57ec77", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:40:18Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "7cba9a3e-4a6f-4d40-8cd2-913e2e57ec77", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:40:18Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "7cba9a3e-4a6f-4d40-8cd2-913e2e57ec77", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:40:19Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "37e32c71-ad06-4f0c-a66d-cd6c40eae71c", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:40:19Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "37e32c71-ad06-4f0c-a66d-cd6c40eae71c", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:40:19Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "37e32c71-ad06-4f0c-a66d-cd6c40eae71c", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:40:19Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "37e32c71-ad06-4f0c-a66d-cd6c40eae71c", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm", "name": "facebook-opt-125m-simulated-kserve-route-authn"} 2026-06-05T14:40:19Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "37e32c71-ad06-4f0c-a66d-cd6c40eae71c", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:40:19Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "37e32c71-ad06-4f0c-a66d-cd6c40eae71c", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:40:19Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "e89e4947-0417-40c3-8c8c-88b79a4a54a7", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-05T14:40:19Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "e89e4947-0417-40c3-8c8c-88b79a4a54a7", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-05T14:40:19Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "e89e4947-0417-40c3-8c8c-88b79a4a54a7", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-05T14:40:19Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "e89e4947-0417-40c3-8c8c-88b79a4a54a7", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm", "name": "premium-simulated-simulated-premium-kserve-route-authn"} 2026-06-05T14:40:19Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "e89e4947-0417-40c3-8c8c-88b79a4a54a7", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:40:19Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "e89e4947-0417-40c3-8c8c-88b79a4a54a7", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-05T14:40:21Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "b02c0d27-859b-49ed-9ece-df798ea0d0cd", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:40:21Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "b02c0d27-859b-49ed-9ece-df798ea0d0cd", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:40:21Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "b02c0d27-859b-49ed-9ece-df798ea0d0cd", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:40:22Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "a6adecf6-9651-40af-98ff-79ce65beb30b", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-05T14:40:22Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "a6adecf6-9651-40af-98ff-79ce65beb30b", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-05T14:40:22Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "a6adecf6-9651-40af-98ff-79ce65beb30b", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-05T14:40:22Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "a6adecf6-9651-40af-98ff-79ce65beb30b", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm", "name": "premium-simulated-simulated-premium-kserve-route-authn"} 2026-06-05T14:40:22Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "a6adecf6-9651-40af-98ff-79ce65beb30b", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:40:22Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "a6adecf6-9651-40af-98ff-79ce65beb30b", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-05T14:40:24Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "cc911aaf-a98e-4bdd-9a1a-419e8cfb5ca8", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-05T14:40:24Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "cc911aaf-a98e-4bdd-9a1a-419e8cfb5ca8", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-05T14:40:24Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "cc911aaf-a98e-4bdd-9a1a-419e8cfb5ca8", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-05T14:40:24Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "cc911aaf-a98e-4bdd-9a1a-419e8cfb5ca8", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm", "name": "e2e-distinct-2-simulated-kserve-route-authn"} 2026-06-05T14:40:24Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "cc911aaf-a98e-4bdd-9a1a-419e8cfb5ca8", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:40:24Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-2-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-2-simulated", "reconcileID": "cc911aaf-a98e-4bdd-9a1a-419e8cfb5ca8", "LLMInferenceService": "e2e-distinct-2-simulated", "namespace": "llm"} 2026-06-05T14:40:43Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "d406a36e-05f3-4181-84df-d6ed9b030fcb", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-05T14:40:43Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "d406a36e-05f3-4181-84df-d6ed9b030fcb", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-05T14:40:43Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "d406a36e-05f3-4181-84df-d6ed9b030fcb", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-05T14:40:43Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "d406a36e-05f3-4181-84df-d6ed9b030fcb", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm", "name": "e2e-distinct-simulated-kserve-route-authn"} 2026-06-05T14:40:43Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "d406a36e-05f3-4181-84df-d6ed9b030fcb", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:40:43Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-distinct-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-distinct-simulated", "reconcileID": "d406a36e-05f3-4181-84df-d6ed9b030fcb", "LLMInferenceService": "e2e-distinct-simulated", "namespace": "llm"} 2026-06-05T14:40:54Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "4249c212-259c-40ac-8284-d9c25c880e66", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-05T14:40:54Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "4249c212-259c-40ac-8284-d9c25c880e66", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-05T14:40:54Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "4249c212-259c-40ac-8284-d9c25c880e66", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-05T14:40:54Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "4249c212-259c-40ac-8284-d9c25c880e66", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm", "name": "e2e-trlp-test-simulated-kserve-route-authn"} 2026-06-05T14:40:54Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "4249c212-259c-40ac-8284-d9c25c880e66", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:40:54Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-trlp-test-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-trlp-test-simulated", "reconcileID": "4249c212-259c-40ac-8284-d9c25c880e66", "LLMInferenceService": "e2e-trlp-test-simulated", "namespace": "llm"} 2026-06-05T14:41:04Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "d7bce903-bc4b-4f35-bcc3-bd1d9422225e", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:41:04Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "d7bce903-bc4b-4f35-bcc3-bd1d9422225e", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:41:04Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "d7bce903-bc4b-4f35-bcc3-bd1d9422225e", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:41:04Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "d7bce903-bc4b-4f35-bcc3-bd1d9422225e", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated-kserve-route-authn"} 2026-06-05T14:41:04Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "d7bce903-bc4b-4f35-bcc3-bd1d9422225e", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:41:04Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "e2e-unconfigured-facebook-opt-125m-simulated", "reconcileID": "d7bce903-bc4b-4f35-bcc3-bd1d9422225e", "LLMInferenceService": "e2e-unconfigured-facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:41:05Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "6112704e-6112-4329-8536-98928129eead", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:41:05Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "6112704e-6112-4329-8536-98928129eead", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:41:05Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "6112704e-6112-4329-8536-98928129eead", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:41:05Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "6112704e-6112-4329-8536-98928129eead", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm", "name": "facebook-opt-125m-simulated-kserve-route-authn"} 2026-06-05T14:41:05Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "6112704e-6112-4329-8536-98928129eead", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:41:05Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"facebook-opt-125m-simulated","namespace":"llm"}, "namespace": "llm", "name": "facebook-opt-125m-simulated", "reconcileID": "6112704e-6112-4329-8536-98928129eead", "LLMInferenceService": "facebook-opt-125m-simulated", "namespace": "llm"} 2026-06-05T14:41:11Z DEBUG Reconciling LLMInferenceService sub-resources {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "01e2ec0d-1cb8-4523-a911-a063e88ba483", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-05T14:41:11Z DEBUG Starting AuthPolicy reconciliation for LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "01e2ec0d-1cb8-4523-a911-a063e88ba483", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-05T14:41:11Z DEBUG Reconciling HTTPRoute AuthPolicy {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "01e2ec0d-1cb8-4523-a911-a063e88ba483", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-05T14:41:11Z DEBUG Processing HTTPRoute AuthPolicy delta {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "01e2ec0d-1cb8-4523-a911-a063e88ba483", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm", "name": "premium-simulated-simulated-premium-kserve-route-authn"} 2026-06-05T14:41:11Z DEBUG AuthType {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "01e2ec0d-1cb8-4523-a911-a063e88ba483", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm", "authType": "userdefined"} 2026-06-05T14:41:11Z DEBUG Successfully reconciled LLMInferenceService {"controller": "llminferenceservice", "controllerGroup": "serving.kserve.io", "controllerKind": "LLMInferenceService", "LLMInferenceService": {"name":"premium-simulated-simulated-premium","namespace":"llm"}, "namespace": "llm", "name": "premium-simulated-simulated-premium", "reconcileID": "01e2ec0d-1cb8-4523-a911-a063e88ba483", "LLMInferenceService": "premium-simulated-simulated-premium", "namespace": "llm"} 2026-06-05T14:41:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "openshift-service-ca.crt", "reconcileID": "65628d4e-48d0-4388-8172-5baeb3e3cf8b", "ConfigMap": "openshift-service-ca.crt", "namespace": "premium-users-namespace"} 2026-06-05T14:41:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "odh-trusted-ca-bundle", "reconcileID": "b3bae914-b3ed-45b9-a8b6-70c1f20021e8", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "premium-users-namespace"} 2026-06-05T14:41:12Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "openshift-service-ca.crt", "reconcileID": "208d5782-2e65-40a2-b322-be498734c342", "ConfigMap": "openshift-service-ca.crt", "namespace": "premium-users-namespace", "create": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:41:12Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "openshift-service-ca.crt", "reconcileID": "208d5782-2e65-40a2-b322-be498734c342", "ConfigMap": "openshift-service-ca.crt", "namespace": "premium-users-namespace"} 2026-06-05T14:41:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "openshift-service-ca.crt", "reconcileID": "54787fcd-debb-4518-9fbe-5e8208675e52", "ConfigMap": "openshift-service-ca.crt", "namespace": "premium-users-namespace"} 2026-06-05T14:41:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "008a314d-206e-44fc-9a0d-b1bf03ae0f65", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "premium-users-namespace"} 2026-06-05T14:41:12Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "odh-trusted-ca-bundle", "reconcileID": "743b2dd0-6abc-46e1-8ffd-8852752073a4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "premium-users-namespace", "update": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:41:12Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "odh-trusted-ca-bundle", "reconcileID": "743b2dd0-6abc-46e1-8ffd-8852752073a4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "premium-users-namespace"} 2026-06-05T14:41:12Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"premium-users-namespace"}, "namespace": "premium-users-namespace", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "54f5e6a7-0c94-43ff-8d8b-7302924957b9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "premium-users-namespace"} 2026-06-05T14:41:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "openshift-service-ca.crt", "reconcileID": "87a33951-9163-4f32-a822-7362fc5ab59d", "ConfigMap": "openshift-service-ca.crt", "namespace": "maas-admin"} 2026-06-05T14:41:13Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "odh-trusted-ca-bundle", "reconcileID": "21f1a82f-8d77-49d4-8370-ea41432fdf5b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "maas-admin", "create": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:41:13Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "odh-trusted-ca-bundle", "reconcileID": "21f1a82f-8d77-49d4-8370-ea41432fdf5b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "maas-admin"} 2026-06-05T14:41:13Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "openshift-service-ca.crt", "reconcileID": "fc214c06-d643-41a0-b680-fb561d32ba15", "ConfigMap": "openshift-service-ca.crt", "namespace": "maas-admin", "create": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:41:13Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "openshift-service-ca.crt", "reconcileID": "fc214c06-d643-41a0-b680-fb561d32ba15", "error": "configmaps \"odh-kserve-custom-ca-bundle\" already exists"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-05T14:41:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a602b874-ebb9-4001-af65-ebefd2f08123", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "maas-admin"} 2026-06-05T14:41:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "openshift-service-ca.crt", "reconcileID": "a9d24bd3-7d19-4b5e-bd98-498b38b79c1b", "ConfigMap": "openshift-service-ca.crt", "namespace": "maas-admin"} 2026-06-05T14:41:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "openshift-service-ca.crt", "reconcileID": "cccf654d-da0f-4eb7-bcfe-442503535d7f", "ConfigMap": "openshift-service-ca.crt", "namespace": "maas-admin"} 2026-06-05T14:41:13Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "odh-trusted-ca-bundle", "reconcileID": "f1fdfed8-55b6-4936-9cf9-4d5cdab0dc66", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "maas-admin", "update": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:41:13Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "odh-trusted-ca-bundle", "reconcileID": "f1fdfed8-55b6-4936-9cf9-4d5cdab0dc66", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "maas-admin"} 2026-06-05T14:41:13Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"maas-admin"}, "namespace": "maas-admin", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "406f7606-ba8f-4b7c-9aaa-361895738f5f", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "maas-admin"} 2026-06-05T14:43:18Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-05T14:43:18Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-05T14:43:21Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "8a3a0ec3-a943-48aa-9827-f67d342ecd73", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-api-unwatched-ns"} 2026-06-05T14:43:21Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-trusted-ca-bundle", "reconcileID": "2485bfee-f8dc-4dd4-8598-a4963a8884e5", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-api-unwatched-ns"} 2026-06-05T14:43:21Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "5a34b2a9-8605-4ad4-aa71-ee3584c51ae7", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-api-unwatched-ns", "create": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:43:21Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "5a34b2a9-8605-4ad4-aa71-ee3584c51ae7", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-api-unwatched-ns"} 2026-06-05T14:43:21Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "fa1b53fa-c370-4c4e-99a9-7e0544471554", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-api-unwatched-ns"} 2026-06-05T14:43:21Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "9177f8af-d634-47f5-ae3d-d62fa7800e46", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-api-unwatched-ns"} 2026-06-05T14:43:21Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-trusted-ca-bundle", "reconcileID": "ad87befd-47e5-4733-a129-532cd2c36946", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-api-unwatched-ns", "update": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:43:21Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-trusted-ca-bundle", "reconcileID": "ad87befd-47e5-4733-a129-532cd2c36946", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-api-unwatched-ns"} 2026-06-05T14:43:21Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "9c6c3c9d-419c-450e-8f67-08bffc677744", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-api-unwatched-ns"} 2026-06-05T14:43:23Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-05T14:43:23Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-05T14:43:35Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "55bb8d5b-8ff9-4b95-9c3e-1ac36b827fbd", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-api-unwatched-ns"} 2026-06-05T14:43:35Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "55bb8d5b-8ff9-4b95-9c3e-1ac36b827fbd", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-api-unwatched-ns", "create": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:43:35Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "55bb8d5b-8ff9-4b95-9c3e-1ac36b827fbd", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-api-unwatched-ns because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-05T14:43:35Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-trusted-ca-bundle", "reconcileID": "fe69c875-d067-4688-8c6e-7b3862530db2", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-api-unwatched-ns"} 2026-06-05T14:43:35Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-trusted-ca-bundle", "reconcileID": "fe69c875-d067-4688-8c6e-7b3862530db2", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-api-unwatched-ns"} 2026-06-05T14:43:35Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "180802a5-c02c-42bd-a600-7ed8d21fa42e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-api-unwatched-ns"} 2026-06-05T14:43:35Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "180802a5-c02c-42bd-a600-7ed8d21fa42e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-api-unwatched-ns"} 2026-06-05T14:43:35Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7072850c-9b65-41c2-a3e5-8e97a5bb8d6a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-api-unwatched-ns"} 2026-06-05T14:43:35Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-api-unwatched-ns"}, "namespace": "e2e-api-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7072850c-9b65-41c2-a3e5-8e97a5bb8d6a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-api-unwatched-ns"} 2026-06-05T14:44:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "d73f1cd8-7b49-46a7-8e0f-a5c408917707", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unwatched-ns"} 2026-06-05T14:44:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-trusted-ca-bundle", "reconcileID": "cf49a447-7644-4fd8-937b-770157f16753", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unwatched-ns"} 2026-06-05T14:44:07Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "b403ffeb-745c-41fa-bf76-35c2231a42f8", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unwatched-ns", "create": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:44:07Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "b403ffeb-745c-41fa-bf76-35c2231a42f8", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unwatched-ns"} 2026-06-05T14:44:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "49d87d10-1e74-4702-ae18-7614a88b7bb4", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unwatched-ns"} 2026-06-05T14:44:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "8dd832fb-2b86-4a3a-9de2-08c63a924667", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unwatched-ns"} 2026-06-05T14:44:07Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-trusted-ca-bundle", "reconcileID": "b90b281b-009d-4ceb-b8f4-c044e1fdba17", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unwatched-ns", "update": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:44:07Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-trusted-ca-bundle", "reconcileID": "b90b281b-009d-4ceb-b8f4-c044e1fdba17", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unwatched-ns"} 2026-06-05T14:44:07Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f3f83296-eec4-49e9-bad8-acc17512f6dd", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unwatched-ns"} 2026-06-05T14:44:37Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "323f57d9-59f4-4b1f-a7a9-4b996a6b6fbc", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unwatched-ns"} 2026-06-05T14:44:37Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "323f57d9-59f4-4b1f-a7a9-4b996a6b6fbc", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unwatched-ns", "create": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:44:37Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "323f57d9-59f4-4b1f-a7a9-4b996a6b6fbc", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-unwatched-ns because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-05T14:44:37Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-trusted-ca-bundle", "reconcileID": "6c60cc83-f110-44f1-a808-d031b728c7fb", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unwatched-ns"} 2026-06-05T14:44:37Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-trusted-ca-bundle", "reconcileID": "6c60cc83-f110-44f1-a808-d031b728c7fb", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-unwatched-ns"} 2026-06-05T14:44:37Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "598607ef-500d-4c69-8c96-36fe719c9dd5", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unwatched-ns"} 2026-06-05T14:44:37Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "openshift-service-ca.crt", "reconcileID": "598607ef-500d-4c69-8c96-36fe719c9dd5", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-unwatched-ns"} 2026-06-05T14:44:37Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c26fe3ce-e3af-446a-ae99-88321bca4b0c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unwatched-ns"} 2026-06-05T14:44:37Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-unwatched-ns"}, "namespace": "e2e-unwatched-ns", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c26fe3ce-e3af-446a-ae99-88321bca4b0c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-unwatched-ns"} 2026-06-05T14:44:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-68b444"}, "namespace": "e2e-modelref-68b444", "name": "openshift-service-ca.crt", "reconcileID": "b1b7b538-e0c6-4fcf-a02b-4d57e13991b0", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-68b444"} 2026-06-05T14:44:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-modelref-68b444"}, "namespace": "e2e-modelref-68b444", "name": "odh-trusted-ca-bundle", "reconcileID": "d6bd1e2f-dd51-4c7f-9e02-91fd4f4561de", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-modelref-68b444"} 2026-06-05T14:44:38Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-68b444"}, "namespace": "e2e-modelref-68b444", "name": "openshift-service-ca.crt", "reconcileID": "0c25758c-a02b-40a8-856d-077ed57acc84", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-68b444", "create": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:44:38Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-68b444"}, "namespace": "e2e-modelref-68b444", "name": "openshift-service-ca.crt", "reconcileID": "0c25758c-a02b-40a8-856d-077ed57acc84", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-68b444"} 2026-06-05T14:44:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-68b444"}, "namespace": "e2e-modelref-68b444", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "7203afed-484a-4067-aec1-d4fa1f6b5127", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-68b444"} 2026-06-05T14:44:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-68b444"}, "namespace": "e2e-modelref-68b444", "name": "openshift-service-ca.crt", "reconcileID": "9912be5f-c7bb-4b3d-957e-14a759095341", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-68b444"} 2026-06-05T14:44:38Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-modelref-68b444"}, "namespace": "e2e-modelref-68b444", "name": "odh-trusted-ca-bundle", "reconcileID": "5b34df66-b9e2-46af-bac0-5b18c43305e4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-modelref-68b444", "update": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:44:38Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-modelref-68b444"}, "namespace": "e2e-modelref-68b444", "name": "odh-trusted-ca-bundle", "reconcileID": "5b34df66-b9e2-46af-bac0-5b18c43305e4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-modelref-68b444"} 2026-06-05T14:44:38Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-68b444"}, "namespace": "e2e-modelref-68b444", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3a34ef83-ff12-4995-952a-28de1e86eeb3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-68b444"} 2026-06-05T14:44:38Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "f9af0cbd-9803-4fa8-a3d3-f2f5c606cf87", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:44:38Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "f9af0cbd-9803-4fa8-a3d3-f2f5c606cf87", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:44:38Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "f9af0cbd-9803-4fa8-a3d3-f2f5c606cf87", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:44:38Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "3d4d1d68-1988-4e8e-a7f5-62eb4a58efcb", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:44:38Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "3d4d1d68-1988-4e8e-a7f5-62eb4a58efcb", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:44:38Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "3d4d1d68-1988-4e8e-a7f5-62eb4a58efcb", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:44:55Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "13987f38-993f-4f0a-8530-b69c45ad30ae", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:44:55Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "13987f38-993f-4f0a-8530-b69c45ad30ae", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:44:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "13987f38-993f-4f0a-8530-b69c45ad30ae", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:44:55Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "00dd07e7-3310-4566-9cb8-349240656cf0", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:44:55Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "00dd07e7-3310-4566-9cb8-349240656cf0", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:44:55Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "00dd07e7-3310-4566-9cb8-349240656cf0", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:45:00Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-68b444"}, "namespace": "e2e-modelref-68b444", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6a7ac5d2-29d9-497a-9b76-0f42418971c2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-68b444"} 2026-06-05T14:45:00Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-68b444"}, "namespace": "e2e-modelref-68b444", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6a7ac5d2-29d9-497a-9b76-0f42418971c2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-68b444", "create": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:45:00Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-68b444"}, "namespace": "e2e-modelref-68b444", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "6a7ac5d2-29d9-497a-9b76-0f42418971c2", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-modelref-68b444 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-05T14:45:00Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-modelref-68b444"}, "namespace": "e2e-modelref-68b444", "name": "odh-trusted-ca-bundle", "reconcileID": "52e4848b-33ba-4eac-90e9-f8203cbb860f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-modelref-68b444"} 2026-06-05T14:45:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-modelref-68b444"}, "namespace": "e2e-modelref-68b444", "name": "odh-trusted-ca-bundle", "reconcileID": "52e4848b-33ba-4eac-90e9-f8203cbb860f", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-modelref-68b444"} 2026-06-05T14:45:00Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-68b444"}, "namespace": "e2e-modelref-68b444", "name": "openshift-service-ca.crt", "reconcileID": "5171bdce-f18c-44cb-9c9a-f698235c1474", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-68b444"} 2026-06-05T14:45:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-68b444"}, "namespace": "e2e-modelref-68b444", "name": "openshift-service-ca.crt", "reconcileID": "5171bdce-f18c-44cb-9c9a-f698235c1474", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-68b444"} 2026-06-05T14:45:00Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-68b444"}, "namespace": "e2e-modelref-68b444", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dc331c34-2394-4be6-bfa3-186b2ec386fe", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-68b444"} 2026-06-05T14:45:00Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-68b444"}, "namespace": "e2e-modelref-68b444", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "dc331c34-2394-4be6-bfa3-186b2ec386fe", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-68b444"} 2026-06-05T14:45:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-d02945"}, "namespace": "e2e-modelref-d02945", "name": "openshift-service-ca.crt", "reconcileID": "2f05effa-2a9e-437b-af9e-545719b39b27", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-d02945"} 2026-06-05T14:45:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-modelref-d02945"}, "namespace": "e2e-modelref-d02945", "name": "odh-trusted-ca-bundle", "reconcileID": "c49826d4-fc05-4375-95f6-df60faba9236", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-modelref-d02945"} 2026-06-05T14:45:08Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-d02945"}, "namespace": "e2e-modelref-d02945", "name": "openshift-service-ca.crt", "reconcileID": "25759ec2-cc53-48bd-be91-29d489ad7fa8", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-d02945", "create": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:45:08Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-d02945"}, "namespace": "e2e-modelref-d02945", "name": "openshift-service-ca.crt", "reconcileID": "25759ec2-cc53-48bd-be91-29d489ad7fa8", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-d02945"} 2026-06-05T14:45:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-d02945"}, "namespace": "e2e-modelref-d02945", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "01dd55f0-7c02-411b-b063-61d6c606b778", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-d02945"} 2026-06-05T14:45:08Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-d02945"}, "namespace": "e2e-modelref-d02945", "name": "openshift-service-ca.crt", "reconcileID": "213fd056-6c63-4b12-97fa-7cebb4402bcd", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-d02945"} 2026-06-05T14:45:09Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "3dfd86cc-c799-460c-8edb-26ca3754ef2f", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:45:09Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "3dfd86cc-c799-460c-8edb-26ca3754ef2f", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:45:09Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "3dfd86cc-c799-460c-8edb-26ca3754ef2f", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:45:09Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "aa4e58c0-1a45-4fb2-8b40-eaf86158c447", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:45:09Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "aa4e58c0-1a45-4fb2-8b40-eaf86158c447", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:45:09Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "aa4e58c0-1a45-4fb2-8b40-eaf86158c447", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:45:09Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-modelref-d02945"}, "namespace": "e2e-modelref-d02945", "name": "odh-trusted-ca-bundle", "reconcileID": "116dfb06-d71b-48a4-9cb2-961b6657171d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-modelref-d02945", "update": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:45:09Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-modelref-d02945"}, "namespace": "e2e-modelref-d02945", "name": "odh-trusted-ca-bundle", "reconcileID": "116dfb06-d71b-48a4-9cb2-961b6657171d", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-modelref-d02945"} 2026-06-05T14:45:09Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-d02945"}, "namespace": "e2e-modelref-d02945", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "21ce2043-96d4-478e-8315-25340234d09b", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-d02945"} 2026-06-05T14:45:25Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0ddaf776-46ad-4a2d-8724-94e7f02eef98", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:45:25Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0ddaf776-46ad-4a2d-8724-94e7f02eef98", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:45:25Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "0ddaf776-46ad-4a2d-8724-94e7f02eef98", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:45:25Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "2ba807a5-05dd-46f6-adb7-4f2338a5aefb", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:45:25Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "2ba807a5-05dd-46f6-adb7-4f2338a5aefb", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T14:45:25Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "2ba807a5-05dd-46f6-adb7-4f2338a5aefb", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T14:45:31Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-d02945"}, "namespace": "e2e-modelref-d02945", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f9e30d76-866b-4192-bd07-c23885ba0ea1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-d02945"} 2026-06-05T14:45:31Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-d02945"}, "namespace": "e2e-modelref-d02945", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f9e30d76-866b-4192-bd07-c23885ba0ea1", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-d02945", "create": "odh-kserve-custom-ca-bundle"} 2026-06-05T14:45:31Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-d02945"}, "namespace": "e2e-modelref-d02945", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f9e30d76-866b-4192-bd07-c23885ba0ea1", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-modelref-d02945 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-05T14:45:31Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-modelref-d02945"}, "namespace": "e2e-modelref-d02945", "name": "odh-trusted-ca-bundle", "reconcileID": "35b897be-9c53-4c96-abfb-e63635a63d96", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-modelref-d02945"} 2026-06-05T14:45:31Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-modelref-d02945"}, "namespace": "e2e-modelref-d02945", "name": "odh-trusted-ca-bundle", "reconcileID": "35b897be-9c53-4c96-abfb-e63635a63d96", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-modelref-d02945"} 2026-06-05T14:45:31Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-d02945"}, "namespace": "e2e-modelref-d02945", "name": "openshift-service-ca.crt", "reconcileID": "b62ad5ef-e635-433d-90af-43c032702e90", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-d02945"} 2026-06-05T14:45:31Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-modelref-d02945"}, "namespace": "e2e-modelref-d02945", "name": "openshift-service-ca.crt", "reconcileID": "b62ad5ef-e635-433d-90af-43c032702e90", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-modelref-d02945"} 2026-06-05T14:45:31Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-d02945"}, "namespace": "e2e-modelref-d02945", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5664b59d-037e-4168-a871-a610a0a4346c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-d02945"} 2026-06-05T14:45:31Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-modelref-d02945"}, "namespace": "e2e-modelref-d02945", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "5664b59d-037e-4168-a871-a610a0a4346c", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-modelref-d02945"} 2026-06-05T14:55:40Z INFO No monitoring namespace detected, skipping monitoring reconciliation. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-05T14:55:40Z INFO Multi Node reconciling. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-05T14:55:41Z INFO ray ca tls secret modified so ray tls cert need to be updated to sync ca.crt {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-05T14:55:41Z INFO Multi Node reconciled successfully. {"ResourceName": "ray-ca-tls", "Namespace": "opendatahub"} 2026-06-05T15:02:49Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "7e864480-a797-4c9c-9b92-eb66367c1efc", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T15:02:49Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "7e864480-a797-4c9c-9b92-eb66367c1efc", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T15:02:49Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "7e864480-a797-4c9c-9b92-eb66367c1efc", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T15:02:54Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "69b9f0b2-8399-4f1a-8268-444c74990c35", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T15:02:54Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "69b9f0b2-8399-4f1a-8268-444c74990c35", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T15:02:54Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "69b9f0b2-8399-4f1a-8268-444c74990c35", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T15:03:18Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "a6ee6c15-12d1-4f00-aad6-df9bb53689f6", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T15:03:18Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "a6ee6c15-12d1-4f00-aad6-df9bb53689f6", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T15:03:18Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "a6ee6c15-12d1-4f00-aad6-df9bb53689f6", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T15:03:31Z DEBUG Reconciling EnvoyFilter for Gateway {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "e521db79-09c1-4b2f-b03f-28c6ff92e5b1", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T15:03:31Z INFO Updating EnvoyFilter {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "e521db79-09c1-4b2f-b03f-28c6ff92e5b1", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress", "name": "maas-default-gateway-authn-ssl"} 2026-06-05T15:03:31Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"maas-default-gateway","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "maas-default-gateway", "reconcileID": "e521db79-09c1-4b2f-b03f-28c6ff92e5b1", "Gateway": "maas-default-gateway", "namespace": "openshift-ingress"} 2026-06-05T15:03:33Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-ff1f5242","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-ff1f5242", "reconcileID": "d2680ae6-b30a-4626-9b6a-6cbb747cf0e9", "Gateway": "e2e-ait-ff1f5242", "namespace": "openshift-ingress"} 2026-06-05T15:03:33Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-ff1f5242","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-ff1f5242", "reconcileID": "e6285be7-6ff2-432d-92b0-db9488f39de7", "Gateway": "e2e-ait-ff1f5242", "namespace": "openshift-ingress"} 2026-06-05T15:03:33Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-ff1f5242","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-ff1f5242", "reconcileID": "cf47fe61-1743-4edf-95cc-6794ab8e31a8", "Gateway": "e2e-ait-ff1f5242", "namespace": "openshift-ingress"} 2026-06-05T15:03:33Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-ff1f5242","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-ff1f5242", "reconcileID": "1441eccf-7866-48d5-a5e9-1ed654d1f0a4", "Gateway": "e2e-ait-ff1f5242", "namespace": "openshift-ingress"} 2026-06-05T15:03:34Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-ff1f5242","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-ff1f5242", "reconcileID": "64e58b5b-fe0d-4e1f-b586-2c5aeceac56b", "Gateway": "e2e-ait-ff1f5242", "namespace": "openshift-ingress"} 2026-06-05T15:03:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-ff1f5242"}, "namespace": "e2e-aitenant-ff1f5242", "name": "openshift-service-ca.crt", "reconcileID": "af48d8de-f47a-435d-8d85-2a6f6e26184f", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-ff1f5242"} 2026-06-05T15:03:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-ff1f5242"}, "namespace": "e2e-aitenant-ff1f5242", "name": "odh-trusted-ca-bundle", "reconcileID": "c828cc73-d193-4afa-a695-90485f8cbb6c", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-ff1f5242"} 2026-06-05T15:03:34Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-ff1f5242"}, "namespace": "e2e-aitenant-ff1f5242", "name": "openshift-service-ca.crt", "reconcileID": "8c402522-57e3-4175-9031-1672cfa91d60", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-ff1f5242", "create": "odh-kserve-custom-ca-bundle"} 2026-06-05T15:03:34Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-ff1f5242"}, "namespace": "e2e-aitenant-ff1f5242", "name": "openshift-service-ca.crt", "reconcileID": "8c402522-57e3-4175-9031-1672cfa91d60", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-ff1f5242"} 2026-06-05T15:03:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-ff1f5242"}, "namespace": "e2e-aitenant-ff1f5242", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "43600e8d-a6ed-4805-9f67-210ba8c2ccc9", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-ff1f5242"} 2026-06-05T15:03:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-ff1f5242"}, "namespace": "e2e-aitenant-ff1f5242", "name": "openshift-service-ca.crt", "reconcileID": "c73019f3-a9b3-4a1c-9d96-9bf6731469ab", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-ff1f5242"} 2026-06-05T15:03:34Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-ff1f5242"}, "namespace": "e2e-aitenant-ff1f5242", "name": "odh-trusted-ca-bundle", "reconcileID": "6c36542e-d957-4711-9d09-32cf5f46d2fe", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-ff1f5242", "update": "odh-kserve-custom-ca-bundle"} 2026-06-05T15:03:34Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-ff1f5242"}, "namespace": "e2e-aitenant-ff1f5242", "name": "odh-trusted-ca-bundle", "reconcileID": "6c36542e-d957-4711-9d09-32cf5f46d2fe", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-ff1f5242"} 2026-06-05T15:03:34Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-ff1f5242"}, "namespace": "e2e-aitenant-ff1f5242", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "ce214148-4738-4a13-b64d-124091d9f702", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-ff1f5242"} 2026-06-05T15:03:35Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-ff1f5242","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-ff1f5242", "reconcileID": "0e64041e-903d-4121-80af-bdfbd25def75", "Gateway": "e2e-ait-ff1f5242", "namespace": "openshift-ingress"} 2026-06-05T15:03:37Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-ff1f5242","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-ff1f5242", "reconcileID": "4709732c-59a6-4843-9d6f-44fe846b564a", "Gateway": "e2e-ait-ff1f5242", "namespace": "openshift-ingress"} 2026-06-05T15:03:46Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-ff1f5242"}, "namespace": "e2e-aitenant-ff1f5242", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3e3351a7-3be2-463b-93bb-7433ac322faf", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-ff1f5242"} 2026-06-05T15:03:46Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-ff1f5242"}, "namespace": "e2e-aitenant-ff1f5242", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3e3351a7-3be2-463b-93bb-7433ac322faf", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-ff1f5242", "create": "odh-kserve-custom-ca-bundle"} 2026-06-05T15:03:46Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-ff1f5242"}, "namespace": "e2e-aitenant-ff1f5242", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3e3351a7-3be2-463b-93bb-7433ac322faf", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-aitenant-ff1f5242 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-05T15:03:46Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-ff1f5242"}, "namespace": "e2e-aitenant-ff1f5242", "name": "odh-trusted-ca-bundle", "reconcileID": "c7760087-2521-4be6-9bdf-806899faf3ef", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-ff1f5242"} 2026-06-05T15:03:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-ff1f5242"}, "namespace": "e2e-aitenant-ff1f5242", "name": "odh-trusted-ca-bundle", "reconcileID": "c7760087-2521-4be6-9bdf-806899faf3ef", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-ff1f5242"} 2026-06-05T15:03:46Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-ff1f5242"}, "namespace": "e2e-aitenant-ff1f5242", "name": "openshift-service-ca.crt", "reconcileID": "af7ac0ba-1033-4a95-9413-562751c8126e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-ff1f5242"} 2026-06-05T15:03:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-ff1f5242"}, "namespace": "e2e-aitenant-ff1f5242", "name": "openshift-service-ca.crt", "reconcileID": "af7ac0ba-1033-4a95-9413-562751c8126e", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-ff1f5242"} 2026-06-05T15:03:46Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-ff1f5242"}, "namespace": "e2e-aitenant-ff1f5242", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b5991b0b-7573-4d88-a1b1-ad5b7a2a65a3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-ff1f5242"} 2026-06-05T15:03:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-ff1f5242"}, "namespace": "e2e-aitenant-ff1f5242", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "b5991b0b-7573-4d88-a1b1-ad5b7a2a65a3", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-ff1f5242"} 2026-06-05T15:03:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-76fd6952","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-76fd6952", "reconcileID": "1f8f782b-cc06-4d07-8366-4af61baacda3", "Gateway": "e2e-ait-76fd6952", "namespace": "openshift-ingress"} 2026-06-05T15:03:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-76fd6952","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-76fd6952", "reconcileID": "172ec94a-b630-42f0-ae2a-623ccb45225e", "Gateway": "e2e-ait-76fd6952", "namespace": "openshift-ingress"} 2026-06-05T15:03:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-76fd6952","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-76fd6952", "reconcileID": "a2156470-f44e-4924-b2a6-55d6f1b55b99", "Gateway": "e2e-ait-76fd6952", "namespace": "openshift-ingress"} 2026-06-05T15:03:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-76fd6952","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-76fd6952", "reconcileID": "8353c57b-b09f-4190-8744-567ebe719c7d", "Gateway": "e2e-ait-76fd6952", "namespace": "openshift-ingress"} 2026-06-05T15:03:46Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-76fd6952","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-76fd6952", "reconcileID": "5f4a4558-5da5-4894-af7a-72a06cdd4a55", "Gateway": "e2e-ait-76fd6952", "namespace": "openshift-ingress"} 2026-06-05T15:03:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-76fd6952"}, "namespace": "e2e-aitenant-76fd6952", "name": "openshift-service-ca.crt", "reconcileID": "da18d0ff-9e51-41d5-bd04-226ba11edff0", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-76fd6952"} 2026-06-05T15:03:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-76fd6952"}, "namespace": "e2e-aitenant-76fd6952", "name": "odh-trusted-ca-bundle", "reconcileID": "9f1807cc-d69a-4fc3-b0bc-9266b7f44c39", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-76fd6952"} 2026-06-05T15:03:46Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-76fd6952"}, "namespace": "e2e-aitenant-76fd6952", "name": "openshift-service-ca.crt", "reconcileID": "13db667b-8f85-41c5-90d7-f67aefb2c085", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-76fd6952", "create": "odh-kserve-custom-ca-bundle"} 2026-06-05T15:03:46Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-76fd6952"}, "namespace": "e2e-aitenant-76fd6952", "name": "openshift-service-ca.crt", "reconcileID": "13db667b-8f85-41c5-90d7-f67aefb2c085", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-76fd6952"} 2026-06-05T15:03:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-76fd6952"}, "namespace": "e2e-aitenant-76fd6952", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a53f1110-0d23-4d73-8f3f-6ee9b6e1f356", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-76fd6952"} 2026-06-05T15:03:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-76fd6952"}, "namespace": "e2e-aitenant-76fd6952", "name": "openshift-service-ca.crt", "reconcileID": "3052b9d2-e0dd-4278-be0e-5dc0998cfb46", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-76fd6952"} 2026-06-05T15:03:46Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-76fd6952"}, "namespace": "e2e-aitenant-76fd6952", "name": "odh-trusted-ca-bundle", "reconcileID": "d0dc9db8-54ca-469e-9698-27d7e8173050", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-76fd6952", "update": "odh-kserve-custom-ca-bundle"} 2026-06-05T15:03:46Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-76fd6952"}, "namespace": "e2e-aitenant-76fd6952", "name": "odh-trusted-ca-bundle", "reconcileID": "d0dc9db8-54ca-469e-9698-27d7e8173050", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-76fd6952"} 2026-06-05T15:03:46Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-76fd6952"}, "namespace": "e2e-aitenant-76fd6952", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "1390c687-6493-42bd-90cb-06a651d136dd", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-76fd6952"} 2026-06-05T15:03:47Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-76fd6952","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-76fd6952", "reconcileID": "c7b6c527-2c60-454f-83be-3272777fcd96", "Gateway": "e2e-ait-76fd6952", "namespace": "openshift-ingress"} 2026-06-05T15:03:50Z DEBUG Successfully reconciled Gateway resources {"controller": "gateway-auth-bootstrap", "controllerGroup": "gateway.networking.k8s.io", "controllerKind": "Gateway", "Gateway": {"name":"e2e-ait-76fd6952","namespace":"openshift-ingress"}, "namespace": "openshift-ingress", "name": "e2e-ait-76fd6952", "reconcileID": "c26ccf87-f3d5-4166-84d8-bbd3dbf2849f", "Gateway": "e2e-ait-76fd6952", "namespace": "openshift-ingress"} 2026-06-05T15:03:59Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-76fd6952"}, "namespace": "e2e-aitenant-76fd6952", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2f07442b-89fc-407c-bf90-81e085b3cb09", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-76fd6952"} 2026-06-05T15:03:59Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-76fd6952"}, "namespace": "e2e-aitenant-76fd6952", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2f07442b-89fc-407c-bf90-81e085b3cb09", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-76fd6952", "create": "odh-kserve-custom-ca-bundle"} 2026-06-05T15:03:59Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-76fd6952"}, "namespace": "e2e-aitenant-76fd6952", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2f07442b-89fc-407c-bf90-81e085b3cb09", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace e2e-aitenant-76fd6952 because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-06-05T15:03:59Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-76fd6952"}, "namespace": "e2e-aitenant-76fd6952", "name": "odh-trusted-ca-bundle", "reconcileID": "90951785-b6c1-489f-a5a6-9f64b74374cf", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-76fd6952"} 2026-06-05T15:03:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"e2e-aitenant-76fd6952"}, "namespace": "e2e-aitenant-76fd6952", "name": "odh-trusted-ca-bundle", "reconcileID": "90951785-b6c1-489f-a5a6-9f64b74374cf", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "e2e-aitenant-76fd6952"} 2026-06-05T15:03:59Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-76fd6952"}, "namespace": "e2e-aitenant-76fd6952", "name": "openshift-service-ca.crt", "reconcileID": "ea487626-a3aa-453a-a06f-c9617e654089", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-76fd6952"} 2026-06-05T15:03:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"e2e-aitenant-76fd6952"}, "namespace": "e2e-aitenant-76fd6952", "name": "openshift-service-ca.crt", "reconcileID": "ea487626-a3aa-453a-a06f-c9617e654089", "ConfigMap": "openshift-service-ca.crt", "namespace": "e2e-aitenant-76fd6952"} 2026-06-05T15:03:59Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-76fd6952"}, "namespace": "e2e-aitenant-76fd6952", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2c4e79a8-fe7f-4cbf-af06-6fa891032615", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-76fd6952"} 2026-06-05T15:03:59Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"e2e-aitenant-76fd6952"}, "namespace": "e2e-aitenant-76fd6952", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "2c4e79a8-fe7f-4cbf-af06-6fa891032615", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "e2e-aitenant-76fd6952"}