{"level":"info","ts":"2026-07-29T01:50:48Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:48Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:48Z","msg":"Validating LLMInferenceService v1alpha2","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:50:48Z","msg":"Validating LLMInferenceService v1alpha2","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:50:48Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:48Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:49Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:49Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:49Z","msg":"Validating LLMInferenceService v1alpha2","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:50:49Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:49Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:49Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:49Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:49Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:49Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:49Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:49Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:49Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:54Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:54Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:54Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:54Z","msg":"Validating LLMInferenceService v1alpha2","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:50:54Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:54Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:54Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:54Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:54Z","msg":"Validating LLMInferenceService v1alpha2","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:50:54Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:54Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:54Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:54Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:54Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:54Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:50:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:50:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:50:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:50:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"29723a12-cb30-486e-bd5c-259a76b371fe","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:14Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"29723a12-cb30-486e-bd5c-259a76b371fe","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"29723a12-cb30-486e-bd5c-259a76b371fe","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"29723a12-cb30-486e-bd5c-259a76b371fe","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"29723a12-cb30-486e-bd5c-259a76b371fe","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"29723a12-cb30-486e-bd5c-259a76b371fe","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Router","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"29723a12-cb30-486e-bd5c-259a76b371fe","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Scheduler","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"29723a12-cb30-486e-bd5c-259a76b371fe","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling HTTPRoute","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"29723a12-cb30-486e-bd5c-259a76b371fe","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"no public URL discovered","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"29723a12-cb30-486e-bd5c-259a76b371fe","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Istio Destination Rules","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"29723a12-cb30-486e-bd5c-259a76b371fe","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Gateway readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"29723a12-cb30-486e-bd5c-259a76b371fe","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","gateway":"openshift-ingress/e2e-ipp-a-295648","ready":true} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileRouter.evaluateGatewayConditions","msg":"All referenced Gateways are ready","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"29723a12-cb30-486e-bd5c-259a76b371fe","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"HTTPRoute readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"29723a12-cb30-486e-bd5c-259a76b371fe","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","route":"ai-tenant-e2e-ipp-a-295648/ipp-route-295648-kserve-route","ready":true} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling Monitoring Resources for LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"29723a12-cb30-486e-bd5c-259a76b371fe","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService metrics reader RBAC for namespace","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"29723a12-cb30-486e-bd5c-259a76b371fe","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService engine monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"29723a12-cb30-486e-bd5c-259a76b371fe","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService scheduler monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"29723a12-cb30-486e-bd5c-259a76b371fe","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"38c4591c-c990-46c7-b201-0baefef78959","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"38c4591c-c990-46c7-b201-0baefef78959","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"38c4591c-c990-46c7-b201-0baefef78959","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"38c4591c-c990-46c7-b201-0baefef78959","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"38c4591c-c990-46c7-b201-0baefef78959","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"38c4591c-c990-46c7-b201-0baefef78959","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Router","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"38c4591c-c990-46c7-b201-0baefef78959","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Scheduler","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"38c4591c-c990-46c7-b201-0baefef78959","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling HTTPRoute","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"38c4591c-c990-46c7-b201-0baefef78959","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"no public URL discovered","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"38c4591c-c990-46c7-b201-0baefef78959","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Istio Destination Rules","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"38c4591c-c990-46c7-b201-0baefef78959","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Gateway readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"38c4591c-c990-46c7-b201-0baefef78959","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","gateway":"openshift-ingress/e2e-ipp-a-295648","ready":true} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileRouter.evaluateGatewayConditions","msg":"All referenced Gateways are ready","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"38c4591c-c990-46c7-b201-0baefef78959","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"HTTPRoute readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"38c4591c-c990-46c7-b201-0baefef78959","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","route":"ai-tenant-e2e-ipp-a-295648/ipp-route-295648-kserve-route","ready":true} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling Monitoring Resources for LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"38c4591c-c990-46c7-b201-0baefef78959","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService metrics reader RBAC for namespace","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"38c4591c-c990-46c7-b201-0baefef78959","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService engine monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"38c4591c-c990-46c7-b201-0baefef78959","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:15Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService scheduler monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"38c4591c-c990-46c7-b201-0baefef78959","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"58e63542-a3b0-4027-a835-a53cc9ca7f74","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"58e63542-a3b0-4027-a835-a53cc9ca7f74","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"58e63542-a3b0-4027-a835-a53cc9ca7f74","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"58e63542-a3b0-4027-a835-a53cc9ca7f74","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"58e63542-a3b0-4027-a835-a53cc9ca7f74","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"58e63542-a3b0-4027-a835-a53cc9ca7f74","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Router","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"58e63542-a3b0-4027-a835-a53cc9ca7f74","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Scheduler","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"58e63542-a3b0-4027-a835-a53cc9ca7f74","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling HTTPRoute","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"58e63542-a3b0-4027-a835-a53cc9ca7f74","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"no public URL discovered","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"58e63542-a3b0-4027-a835-a53cc9ca7f74","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Istio Destination Rules","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"58e63542-a3b0-4027-a835-a53cc9ca7f74","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Gateway readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"58e63542-a3b0-4027-a835-a53cc9ca7f74","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","gateway":"openshift-ingress/e2e-ipp-a-295648","ready":true} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile.reconcileRouter.evaluateGatewayConditions","msg":"All referenced Gateways are ready","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"58e63542-a3b0-4027-a835-a53cc9ca7f74","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"HTTPRoute readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"58e63542-a3b0-4027-a835-a53cc9ca7f74","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","route":"ai-tenant-e2e-ipp-a-295648/ipp-route-295648-kserve-route","ready":true} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling Monitoring Resources for LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"58e63542-a3b0-4027-a835-a53cc9ca7f74","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService metrics reader RBAC for namespace","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"58e63542-a3b0-4027-a835-a53cc9ca7f74","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService engine monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"58e63542-a3b0-4027-a835-a53cc9ca7f74","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService scheduler monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"58e63542-a3b0-4027-a835-a53cc9ca7f74","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"46b13827-23ba-448e-b122-4cbdff5bd024","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"46b13827-23ba-448e-b122-4cbdff5bd024","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"46b13827-23ba-448e-b122-4cbdff5bd024","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"46b13827-23ba-448e-b122-4cbdff5bd024","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"46b13827-23ba-448e-b122-4cbdff5bd024","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"46b13827-23ba-448e-b122-4cbdff5bd024","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Router","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"46b13827-23ba-448e-b122-4cbdff5bd024","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Scheduler","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"46b13827-23ba-448e-b122-4cbdff5bd024","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling HTTPRoute","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"46b13827-23ba-448e-b122-4cbdff5bd024","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:19Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:19Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"no public URL discovered","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"46b13827-23ba-448e-b122-4cbdff5bd024","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Istio Destination Rules","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"46b13827-23ba-448e-b122-4cbdff5bd024","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Gateway readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"46b13827-23ba-448e-b122-4cbdff5bd024","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","gateway":"openshift-ingress/e2e-ipp-a-295648","ready":true} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileRouter.evaluateGatewayConditions","msg":"All referenced Gateways are ready","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"46b13827-23ba-448e-b122-4cbdff5bd024","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"HTTPRoute readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"46b13827-23ba-448e-b122-4cbdff5bd024","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","route":"ai-tenant-e2e-ipp-a-295648/ipp-route-295648-kserve-route","ready":true} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling Monitoring Resources for LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"46b13827-23ba-448e-b122-4cbdff5bd024","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService metrics reader RBAC for namespace","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"46b13827-23ba-448e-b122-4cbdff5bd024","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService engine monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"46b13827-23ba-448e-b122-4cbdff5bd024","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService scheduler monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"46b13827-23ba-448e-b122-4cbdff5bd024","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"96e2dd9f-a5f7-4c01-99eb-721e0477bf5d","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"96e2dd9f-a5f7-4c01-99eb-721e0477bf5d","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"96e2dd9f-a5f7-4c01-99eb-721e0477bf5d","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"96e2dd9f-a5f7-4c01-99eb-721e0477bf5d","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"96e2dd9f-a5f7-4c01-99eb-721e0477bf5d","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"96e2dd9f-a5f7-4c01-99eb-721e0477bf5d","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Router","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"96e2dd9f-a5f7-4c01-99eb-721e0477bf5d","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Scheduler","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"96e2dd9f-a5f7-4c01-99eb-721e0477bf5d","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling HTTPRoute","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"96e2dd9f-a5f7-4c01-99eb-721e0477bf5d","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"no public URL discovered","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"96e2dd9f-a5f7-4c01-99eb-721e0477bf5d","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Istio Destination Rules","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"96e2dd9f-a5f7-4c01-99eb-721e0477bf5d","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Gateway readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"96e2dd9f-a5f7-4c01-99eb-721e0477bf5d","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","gateway":"openshift-ingress/e2e-ipp-a-295648","ready":true} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileRouter.evaluateGatewayConditions","msg":"All referenced Gateways are ready","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"96e2dd9f-a5f7-4c01-99eb-721e0477bf5d","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"HTTPRoute readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"96e2dd9f-a5f7-4c01-99eb-721e0477bf5d","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","route":"ai-tenant-e2e-ipp-a-295648/ipp-route-295648-kserve-route","ready":true} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling Monitoring Resources for LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"96e2dd9f-a5f7-4c01-99eb-721e0477bf5d","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService metrics reader RBAC for namespace","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"96e2dd9f-a5f7-4c01-99eb-721e0477bf5d","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService engine monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"96e2dd9f-a5f7-4c01-99eb-721e0477bf5d","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:20Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService scheduler monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"96e2dd9f-a5f7-4c01-99eb-721e0477bf5d","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:55Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"87a3576c-d2cc-4b24-9d62-e6d63f3853d0","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:55Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:55Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"87a3576c-d2cc-4b24-9d62-e6d63f3853d0","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:55Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"87a3576c-d2cc-4b24-9d62-e6d63f3853d0","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:55Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"87a3576c-d2cc-4b24-9d62-e6d63f3853d0","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:55Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"87a3576c-d2cc-4b24-9d62-e6d63f3853d0","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:55Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"87a3576c-d2cc-4b24-9d62-e6d63f3853d0","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:55Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:56Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"error","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService","msg":"Failed to reconcile LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"87a3576c-d2cc-4b24-9d62-e6d63f3853d0","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"failed to reconcile workload: failed to reconcile workload service: failed to create v1.Service ai-tenant-e2e-ipp-a-295648/ipp-route-295648-kserve-workload-svc: services \"ipp-route-295648-kserve-workload-svc\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ipp-a-295648 because it is being terminated","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:232\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"info","ts":"2026-07-29T01:51:56Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:56Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:56Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:56Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:56Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:56Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"error","ts":"2026-07-29T01:51:56Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"87a3576c-d2cc-4b24-9d62-e6d63f3853d0","error":"failed to reconcile workload: failed to reconcile workload service: failed to create v1.Service ai-tenant-e2e-ipp-a-295648/ipp-route-295648-kserve-workload-svc: services \"ipp-route-295648-kserve-workload-svc\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ipp-a-295648 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"9f4f19ec-d959-4b23-86a3-3f39ef3ace81","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"9f4f19ec-d959-4b23-86a3-3f39ef3ace81","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"9f4f19ec-d959-4b23-86a3-3f39ef3ace81","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"9f4f19ec-d959-4b23-86a3-3f39ef3ace81","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"9f4f19ec-d959-4b23-86a3-3f39ef3ace81","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"9f4f19ec-d959-4b23-86a3-3f39ef3ace81","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"error","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService","msg":"Failed to reconcile LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"9f4f19ec-d959-4b23-86a3-3f39ef3ace81","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"failed to reconcile workload: failed to reconcile workload service: failed to create v1.Service ai-tenant-e2e-ipp-a-295648/ipp-route-295648-kserve-workload-svc: services \"ipp-route-295648-kserve-workload-svc\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ipp-a-295648 because it is being terminated","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:232\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"error","ts":"2026-07-29T01:51:56Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"9f4f19ec-d959-4b23-86a3-3f39ef3ace81","error":"failed to reconcile workload: failed to reconcile workload service: failed to create v1.Service ai-tenant-e2e-ipp-a-295648/ipp-route-295648-kserve-workload-svc: services \"ipp-route-295648-kserve-workload-svc\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ipp-a-295648 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"f293949c-82ac-4a64-8f3f-a9416816fcbd","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"f293949c-82ac-4a64-8f3f-a9416816fcbd","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"f293949c-82ac-4a64-8f3f-a9416816fcbd","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"f293949c-82ac-4a64-8f3f-a9416816fcbd","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"f293949c-82ac-4a64-8f3f-a9416816fcbd","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"f293949c-82ac-4a64-8f3f-a9416816fcbd","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"error","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService","msg":"Failed to reconcile LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"f293949c-82ac-4a64-8f3f-a9416816fcbd","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"failed to reconcile workload: failed to reconcile workload service: failed to create v1.Service ai-tenant-e2e-ipp-a-295648/ipp-route-295648-kserve-workload-svc: services \"ipp-route-295648-kserve-workload-svc\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ipp-a-295648 because it is being terminated","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:232\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"error","ts":"2026-07-29T01:51:56Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"f293949c-82ac-4a64-8f3f-a9416816fcbd","error":"failed to reconcile workload: failed to reconcile workload service: failed to create v1.Service ai-tenant-e2e-ipp-a-295648/ipp-route-295648-kserve-workload-svc: services \"ipp-route-295648-kserve-workload-svc\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ipp-a-295648 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"dcf2852a-3e1f-4161-8fe8-ec52f87ef960","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"dcf2852a-3e1f-4161-8fe8-ec52f87ef960","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"dcf2852a-3e1f-4161-8fe8-ec52f87ef960","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"dcf2852a-3e1f-4161-8fe8-ec52f87ef960","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"dcf2852a-3e1f-4161-8fe8-ec52f87ef960","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"dcf2852a-3e1f-4161-8fe8-ec52f87ef960","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"error","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService","msg":"Failed to reconcile LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"dcf2852a-3e1f-4161-8fe8-ec52f87ef960","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-ipp-a-295648/ipp-route-295648-kserve: deployments.apps \"ipp-route-295648-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ipp-a-295648 because it is being terminated","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:232\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"error","ts":"2026-07-29T01:51:56Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"dcf2852a-3e1f-4161-8fe8-ec52f87ef960","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-ipp-a-295648/ipp-route-295648-kserve: deployments.apps \"ipp-route-295648-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ipp-a-295648 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ecb5c1b4-fd53-4362-bbc0-2c4a5ac55d2e","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ecb5c1b4-fd53-4362-bbc0-2c4a5ac55d2e","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ecb5c1b4-fd53-4362-bbc0-2c4a5ac55d2e","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ecb5c1b4-fd53-4362-bbc0-2c4a5ac55d2e","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"error","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ecb5c1b4-fd53-4362-bbc0-2c4a5ac55d2e","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:64\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"error","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ecb5c1b4-fd53-4362-bbc0-2c4a5ac55d2e","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:68\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ecb5c1b4-fd53-4362-bbc0-2c4a5ac55d2e","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"error","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ecb5c1b4-fd53-4362-bbc0-2c4a5ac55d2e","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:374\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:44\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"error","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ecb5c1b4-fd53-4362-bbc0-2c4a5ac55d2e","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:395\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:47\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ecb5c1b4-fd53-4362-bbc0-2c4a5ac55d2e","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"error","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ecb5c1b4-fd53-4362-bbc0-2c4a5ac55d2e","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:424\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:336\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:45\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:85\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"error","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService","msg":"Failed to reconcile LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ecb5c1b4-fd53-4362-bbc0-2c4a5ac55d2e","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-ipp-a-295648/ipp-route-295648-kserve: deployments.apps \"ipp-route-295648-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ipp-a-295648 because it is being terminated","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:232\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"error","ts":"2026-07-29T01:51:56Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ecb5c1b4-fd53-4362-bbc0-2c4a5ac55d2e","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-ipp-a-295648/ipp-route-295648-kserve: deployments.apps \"ipp-route-295648-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ipp-a-295648 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"6a8b5940-1ad1-478a-9167-7de798106246","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"6a8b5940-1ad1-478a-9167-7de798106246","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"6a8b5940-1ad1-478a-9167-7de798106246","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"6a8b5940-1ad1-478a-9167-7de798106246","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"error","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"6a8b5940-1ad1-478a-9167-7de798106246","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:64\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"error","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"6a8b5940-1ad1-478a-9167-7de798106246","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:68\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"6a8b5940-1ad1-478a-9167-7de798106246","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"error","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"6a8b5940-1ad1-478a-9167-7de798106246","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:374\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:44\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"error","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"6a8b5940-1ad1-478a-9167-7de798106246","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:395\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:47\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"6a8b5940-1ad1-478a-9167-7de798106246","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"error","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"6a8b5940-1ad1-478a-9167-7de798106246","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:424\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:336\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:45\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:85\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"error","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService","msg":"Failed to reconcile LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"6a8b5940-1ad1-478a-9167-7de798106246","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-ipp-a-295648/ipp-route-295648-kserve: deployments.apps \"ipp-route-295648-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ipp-a-295648 because it is being terminated","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:232\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"error","ts":"2026-07-29T01:51:56Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"6a8b5940-1ad1-478a-9167-7de798106246","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-ipp-a-295648/ipp-route-295648-kserve: deployments.apps \"ipp-route-295648-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ipp-a-295648 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ef1ad7e0-d09f-43e2-8491-4e67d1f7c8f9","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ef1ad7e0-d09f-43e2-8491-4e67d1f7c8f9","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ef1ad7e0-d09f-43e2-8491-4e67d1f7c8f9","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ef1ad7e0-d09f-43e2-8491-4e67d1f7c8f9","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"error","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ef1ad7e0-d09f-43e2-8491-4e67d1f7c8f9","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:64\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"error","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ef1ad7e0-d09f-43e2-8491-4e67d1f7c8f9","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:68\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ef1ad7e0-d09f-43e2-8491-4e67d1f7c8f9","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"error","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ef1ad7e0-d09f-43e2-8491-4e67d1f7c8f9","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:374\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:44\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"error","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ef1ad7e0-d09f-43e2-8491-4e67d1f7c8f9","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:395\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:47\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ef1ad7e0-d09f-43e2-8491-4e67d1f7c8f9","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"error","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ef1ad7e0-d09f-43e2-8491-4e67d1f7c8f9","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:424\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:336\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:45\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:85\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"error","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService","msg":"Failed to reconcile LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ef1ad7e0-d09f-43e2-8491-4e67d1f7c8f9","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-ipp-a-295648/ipp-route-295648-kserve: deployments.apps \"ipp-route-295648-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ipp-a-295648 because it is being terminated","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:232\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"error","ts":"2026-07-29T01:51:56Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"ef1ad7e0-d09f-43e2-8491-4e67d1f7c8f9","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-ipp-a-295648/ipp-route-295648-kserve: deployments.apps \"ipp-route-295648-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ipp-a-295648 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"b8610745-1423-4c20-b3f6-7942b50fbd18","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"b8610745-1423-4c20-b3f6-7942b50fbd18","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"b8610745-1423-4c20-b3f6-7942b50fbd18","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:56Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"b8610745-1423-4c20-b3f6-7942b50fbd18","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"error","ts":"2026-07-29T01:51:57Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"b8610745-1423-4c20-b3f6-7942b50fbd18","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:64\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"error","ts":"2026-07-29T01:51:57Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"b8610745-1423-4c20-b3f6-7942b50fbd18","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:68\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"info","ts":"2026-07-29T01:51:57Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"b8610745-1423-4c20-b3f6-7942b50fbd18","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"error","ts":"2026-07-29T01:51:57Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"b8610745-1423-4c20-b3f6-7942b50fbd18","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:374\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:44\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"error","ts":"2026-07-29T01:51:57Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"b8610745-1423-4c20-b3f6-7942b50fbd18","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:395\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:47\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"info","ts":"2026-07-29T01:51:57Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"b8610745-1423-4c20-b3f6-7942b50fbd18","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"error","ts":"2026-07-29T01:51:57Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"b8610745-1423-4c20-b3f6-7942b50fbd18","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:424\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:336\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:45\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:85\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"error","ts":"2026-07-29T01:51:57Z","logger":"LLMInferenceService","msg":"Failed to reconcile LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"b8610745-1423-4c20-b3f6-7942b50fbd18","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-ipp-a-295648/ipp-route-295648-kserve: deployments.apps \"ipp-route-295648-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ipp-a-295648 because it is being terminated","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:232\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"error","ts":"2026-07-29T01:51:57Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"b8610745-1423-4c20-b3f6-7942b50fbd18","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-ipp-a-295648/ipp-route-295648-kserve: deployments.apps \"ipp-route-295648-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-ipp-a-295648 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"info","ts":"2026-07-29T01:51:57Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"e5613d7e-0a57-4b41-8bdc-c8ee2cd3f4b5","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:57Z","logger":"LLMInferenceService","msg":"Marked for deletion, finalizing resources","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"e5613d7e-0a57-4b41-8bdc-c8ee2cd3f4b5","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"error","ts":"2026-07-29T01:51:57Z","logger":"LLMInferenceService","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"e5613d7e-0a57-4b41-8bdc-c8ee2cd3f4b5","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSchedulerServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/scheduler.go:666\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSchedulerServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/scheduler.go:137\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).finalize\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:286\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:201\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"info","ts":"2026-07-29T01:51:57Z","logger":"LLMInferenceService.cleanupMonitoring","msg":"Cleaning up monitoring resources - last LLMInferenceService in namespace","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"e5613d7e-0a57-4b41-8bdc-c8ee2cd3f4b5","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"info","ts":"2026-07-29T01:51:57Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:57Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:57Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:57Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:57Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:57Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:57Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:57Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:57Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:57Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:57Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:57Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:57Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:57Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:57Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:57Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:57Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:57Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:51:57Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"f38b6674-a93f-4609-aa31-321599c48c81","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:57Z","logger":"LLMInferenceService","msg":"Marked for deletion, finalizing resources","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"f38b6674-a93f-4609-aa31-321599c48c81","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"error","ts":"2026-07-29T01:51:57Z","logger":"LLMInferenceService","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"f38b6674-a93f-4609-aa31-321599c48c81","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSchedulerServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/scheduler.go:666\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSchedulerServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/scheduler.go:137\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).finalize\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:286\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:201\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"info","ts":"2026-07-29T01:51:57Z","logger":"LLMInferenceService.cleanupMonitoring","msg":"Cleaning up monitoring resources - last LLMInferenceService in namespace","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"f38b6674-a93f-4609-aa31-321599c48c81","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"} {"level":"error","ts":"2026-07-29T01:51:57Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"f38b6674-a93f-4609-aa31-321599c48c81","error":"llminferenceservices.serving.kserve.io \"ipp-route-295648\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"} {"level":"info","ts":"2026-07-29T01:51:57Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"dc7f9598-3d8a-44ff-a676-6f605fd4bc69","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:51:57Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"ipp-route-295648","namespace":"ai-tenant-e2e-ipp-a-295648"},"namespace":"ai-tenant-e2e-ipp-a-295648","name":"ipp-route-295648","reconcileID":"76a51d3c-0d33-42a4-9a57-f60d85295291","Namespace":"ai-tenant-e2e-ipp-a-295648","Name":"ipp-route-295648"} {"level":"info","ts":"2026-07-29T01:52:03Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:03Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:03Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:03Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:03Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:03Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:14Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:15Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:18Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:18Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:18Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:18Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:18Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:18Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:18Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:18Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:19Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:19Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:19Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:19Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:20Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:31Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:31Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:31Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:31Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:31Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:31Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:31Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:31Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:36Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:36Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:36Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:36Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:36Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:36Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:36Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:36Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:36Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:36Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:36Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:36Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:38Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:38Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:38Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:38Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:38Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:38Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:49Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:49Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:49Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:49Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:49Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:49Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:49Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:49Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:49Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:49Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:49Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:50Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:50Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:50Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:50Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:50Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:50Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:50Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:50Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:50Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:50Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:50Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:50Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:50Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:50Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:50Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:50Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:50Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:50Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:50Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:53Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:53Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:53Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:53Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:53Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:53Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:53Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:53Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:53Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:53Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:53Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:53Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:56Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:56Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:56Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:56Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:56Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"} {"level":"info","ts":"2026-07-29T01:52:56Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"}