{"level":"info","ts":"2026-07-06T19:50:48Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"4f277448-0bd8-45b1-a102-80cbd647efc9","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:50:48Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:50:48Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"4f277448-0bd8-45b1-a102-80cbd647efc9","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:50:48Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"4f277448-0bd8-45b1-a102-80cbd647efc9","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:50:48Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"4f277448-0bd8-45b1-a102-80cbd647efc9","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:50:48Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Creating CA-signed certificate","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"4f277448-0bd8-45b1-a102-80cbd647efc9","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","ips":["10.133.0.59","127.0.0.1","172.31.253.20"],"dnsNames":["localhost","rate-test-model-9293de-kserve-workload-svc.ai-tenant-e2e-shared-b-9293de.svc","rate-test-model-9293de-kserve-workload-svc.ai-tenant-e2e-shared-b-9293de.svc.cluster.local"]}
{"level":"info","ts":"2026-07-06T19:50:57Z","logger":"admission","msg":"Validating LLMInferenceServiceConfig v1alpha2","webhookGroup":"serving.kserve.io","webhookKind":"LLMInferenceServiceConfig","LLMInferenceServiceConfig":{"name":"v3-5-0-ea-2-kserve-config-llm-decode-template","namespace":"opendatahub"},"namespace":"opendatahub","name":"v3-5-0-ea-2-kserve-config-llm-decode-template","resource":{"group":"serving.kserve.io","version":"v1alpha2","resource":"llminferenceserviceconfigs"},"user":"system:serviceaccount:opendatahub:opendatahub-operator-controller-manager","requestID":"d4d05139-f8aa-4d5a-916a-ee62d1e50484","name":"v3-5-0-ea-2-kserve-config-llm-decode-template","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-06T19:50:57Z","logger":"admission","msg":"Validating LLMInferenceServiceConfig v1alpha2","webhookGroup":"serving.kserve.io","webhookKind":"LLMInferenceServiceConfig","LLMInferenceServiceConfig":{"name":"v3-5-0-ea-2-kserve-config-llm-decode-worker-data-parallel","namespace":"opendatahub"},"namespace":"opendatahub","name":"v3-5-0-ea-2-kserve-config-llm-decode-worker-data-parallel","resource":{"group":"serving.kserve.io","version":"v1alpha2","resource":"llminferenceserviceconfigs"},"user":"system:serviceaccount:opendatahub:opendatahub-operator-controller-manager","requestID":"7309edaf-c614-46cf-ab6c-8b0f4dce035a","name":"v3-5-0-ea-2-kserve-config-llm-decode-worker-data-parallel","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-06T19:50:57Z","logger":"admission","msg":"Validating LLMInferenceServiceConfig v1alpha2","webhookGroup":"serving.kserve.io","webhookKind":"LLMInferenceServiceConfig","LLMInferenceServiceConfig":{"name":"v3-5-0-ea-2-kserve-config-llm-prefill-template","namespace":"opendatahub"},"namespace":"opendatahub","name":"v3-5-0-ea-2-kserve-config-llm-prefill-template","resource":{"group":"serving.kserve.io","version":"v1alpha2","resource":"llminferenceserviceconfigs"},"user":"system:serviceaccount:opendatahub:opendatahub-operator-controller-manager","requestID":"4fc696f5-a345-496e-a012-1da639330567","name":"v3-5-0-ea-2-kserve-config-llm-prefill-template","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-06T19:50:57Z","logger":"admission","msg":"Validating LLMInferenceServiceConfig v1alpha2","webhookGroup":"serving.kserve.io","webhookKind":"LLMInferenceServiceConfig","LLMInferenceServiceConfig":{"name":"v3-5-0-ea-2-kserve-config-llm-prefill-worker-data-parallel","namespace":"opendatahub"},"namespace":"opendatahub","name":"v3-5-0-ea-2-kserve-config-llm-prefill-worker-data-parallel","resource":{"group":"serving.kserve.io","version":"v1alpha2","resource":"llminferenceserviceconfigs"},"user":"system:serviceaccount:opendatahub:opendatahub-operator-controller-manager","requestID":"d51ebd56-b976-4dda-a0e1-214aa10aa3ca","name":"v3-5-0-ea-2-kserve-config-llm-prefill-worker-data-parallel","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-06T19:50:57Z","logger":"admission","msg":"Validating LLMInferenceServiceConfig v1alpha2","webhookGroup":"serving.kserve.io","webhookKind":"LLMInferenceServiceConfig","LLMInferenceServiceConfig":{"name":"v3-5-0-ea-2-kserve-config-llm-router-route","namespace":"opendatahub"},"namespace":"opendatahub","name":"v3-5-0-ea-2-kserve-config-llm-router-route","resource":{"group":"serving.kserve.io","version":"v1alpha2","resource":"llminferenceserviceconfigs"},"user":"system:serviceaccount:opendatahub:opendatahub-operator-controller-manager","requestID":"63a9a894-46af-4321-bae9-a4723b78c1fb","name":"v3-5-0-ea-2-kserve-config-llm-router-route","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-06T19:50:58Z","logger":"admission","msg":"Validating LLMInferenceServiceConfig v1alpha2","webhookGroup":"serving.kserve.io","webhookKind":"LLMInferenceServiceConfig","LLMInferenceServiceConfig":{"name":"v3-5-0-ea-2-kserve-config-llm-scheduler","namespace":"opendatahub"},"namespace":"opendatahub","name":"v3-5-0-ea-2-kserve-config-llm-scheduler","resource":{"group":"serving.kserve.io","version":"v1alpha2","resource":"llminferenceserviceconfigs"},"user":"system:serviceaccount:opendatahub:opendatahub-operator-controller-manager","requestID":"c65cc1b6-206c-41ad-b11e-f36f5eabcedb","name":"v3-5-0-ea-2-kserve-config-llm-scheduler","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-06T19:50:58Z","logger":"admission","msg":"Validating LLMInferenceServiceConfig v1alpha2","webhookGroup":"serving.kserve.io","webhookKind":"LLMInferenceServiceConfig","LLMInferenceServiceConfig":{"name":"v3-5-0-ea-2-kserve-config-llm-template","namespace":"opendatahub"},"namespace":"opendatahub","name":"v3-5-0-ea-2-kserve-config-llm-template","resource":{"group":"serving.kserve.io","version":"v1alpha2","resource":"llminferenceserviceconfigs"},"user":"system:serviceaccount:opendatahub:opendatahub-operator-controller-manager","requestID":"0282e879-8734-4e0f-9d16-31bf6b83b4ed","name":"v3-5-0-ea-2-kserve-config-llm-template","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-06T19:50:58Z","logger":"admission","msg":"Validating LLMInferenceServiceConfig v1alpha2","webhookGroup":"serving.kserve.io","webhookKind":"LLMInferenceServiceConfig","LLMInferenceServiceConfig":{"name":"v3-5-0-ea-2-kserve-config-llm-template-amd-rocm","namespace":"opendatahub"},"namespace":"opendatahub","name":"v3-5-0-ea-2-kserve-config-llm-template-amd-rocm","resource":{"group":"serving.kserve.io","version":"v1alpha2","resource":"llminferenceserviceconfigs"},"user":"system:serviceaccount:opendatahub:opendatahub-operator-controller-manager","requestID":"ebe4f09a-33c5-4f40-bc15-76007c8be98d","name":"v3-5-0-ea-2-kserve-config-llm-template-amd-rocm","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-06T19:50:58Z","logger":"admission","msg":"Validating LLMInferenceServiceConfig v1alpha2","webhookGroup":"serving.kserve.io","webhookKind":"LLMInferenceServiceConfig","LLMInferenceServiceConfig":{"name":"v3-5-0-ea-2-kserve-config-llm-template-ibm-spyre-ppc64le","namespace":"opendatahub"},"namespace":"opendatahub","name":"v3-5-0-ea-2-kserve-config-llm-template-ibm-spyre-ppc64le","resource":{"group":"serving.kserve.io","version":"v1alpha2","resource":"llminferenceserviceconfigs"},"user":"system:serviceaccount:opendatahub:opendatahub-operator-controller-manager","requestID":"d20cea40-d324-4630-b98b-8176857c0a9e","name":"v3-5-0-ea-2-kserve-config-llm-template-ibm-spyre-ppc64le","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-06T19:50:58Z","logger":"admission","msg":"Validating LLMInferenceServiceConfig v1alpha2","webhookGroup":"serving.kserve.io","webhookKind":"LLMInferenceServiceConfig","LLMInferenceServiceConfig":{"name":"v3-5-0-ea-2-kserve-config-llm-template-ibm-spyre-s390x","namespace":"opendatahub"},"namespace":"opendatahub","name":"v3-5-0-ea-2-kserve-config-llm-template-ibm-spyre-s390x","resource":{"group":"serving.kserve.io","version":"v1alpha2","resource":"llminferenceserviceconfigs"},"user":"system:serviceaccount:opendatahub:opendatahub-operator-controller-manager","requestID":"d8428340-dc3d-472f-8ca2-8dc08f39164f","name":"v3-5-0-ea-2-kserve-config-llm-template-ibm-spyre-s390x","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-06T19:50:58Z","logger":"admission","msg":"Validating LLMInferenceServiceConfig v1alpha2","webhookGroup":"serving.kserve.io","webhookKind":"LLMInferenceServiceConfig","LLMInferenceServiceConfig":{"name":"v3-5-0-ea-2-kserve-config-llm-template-ibm-spyre-x86","namespace":"opendatahub"},"namespace":"opendatahub","name":"v3-5-0-ea-2-kserve-config-llm-template-ibm-spyre-x86","resource":{"group":"serving.kserve.io","version":"v1alpha2","resource":"llminferenceserviceconfigs"},"user":"system:serviceaccount:opendatahub:opendatahub-operator-controller-manager","requestID":"07937357-5c51-4b35-9039-6ddc791d74fc","name":"v3-5-0-ea-2-kserve-config-llm-template-ibm-spyre-x86","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-06T19:50:58Z","logger":"admission","msg":"Validating LLMInferenceServiceConfig v1alpha2","webhookGroup":"serving.kserve.io","webhookKind":"LLMInferenceServiceConfig","LLMInferenceServiceConfig":{"name":"v3-5-0-ea-2-kserve-config-llm-template-intel-gaudi","namespace":"opendatahub"},"namespace":"opendatahub","name":"v3-5-0-ea-2-kserve-config-llm-template-intel-gaudi","resource":{"group":"serving.kserve.io","version":"v1alpha2","resource":"llminferenceserviceconfigs"},"user":"system:serviceaccount:opendatahub:opendatahub-operator-controller-manager","requestID":"05979cf0-b72d-486a-a27b-725720e332a0","name":"v3-5-0-ea-2-kserve-config-llm-template-intel-gaudi","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-06T19:50:58Z","logger":"admission","msg":"Validating LLMInferenceServiceConfig v1alpha2","webhookGroup":"serving.kserve.io","webhookKind":"LLMInferenceServiceConfig","LLMInferenceServiceConfig":{"name":"v3-5-0-ea-2-kserve-config-llm-template-nvidia-cuda","namespace":"opendatahub"},"namespace":"opendatahub","name":"v3-5-0-ea-2-kserve-config-llm-template-nvidia-cuda","resource":{"group":"serving.kserve.io","version":"v1alpha2","resource":"llminferenceserviceconfigs"},"user":"system:serviceaccount:opendatahub:opendatahub-operator-controller-manager","requestID":"5a06ba52-6de6-4d60-9da1-14d333220de8","name":"v3-5-0-ea-2-kserve-config-llm-template-nvidia-cuda","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-06T19:50:58Z","logger":"admission","msg":"Validating LLMInferenceServiceConfig v1alpha2","webhookGroup":"serving.kserve.io","webhookKind":"LLMInferenceServiceConfig","LLMInferenceServiceConfig":{"name":"v3-5-0-ea-2-kserve-config-llm-worker-data-parallel","namespace":"opendatahub"},"namespace":"opendatahub","name":"v3-5-0-ea-2-kserve-config-llm-worker-data-parallel","resource":{"group":"serving.kserve.io","version":"v1alpha2","resource":"llminferenceserviceconfigs"},"user":"system:serviceaccount:opendatahub:opendatahub-operator-controller-manager","requestID":"3915fc09-f4e9-4334-9bca-9a31c1da255f","name":"v3-5-0-ea-2-kserve-config-llm-worker-data-parallel","namespace":"opendatahub"}
{"level":"info","ts":"2026-07-06T19:51:01Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"4f277448-0bd8-45b1-a102-80cbd647efc9","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:01Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"4f277448-0bd8-45b1-a102-80cbd647efc9","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:01Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Router","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"4f277448-0bd8-45b1-a102-80cbd647efc9","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:01Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Scheduler","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"4f277448-0bd8-45b1-a102-80cbd647efc9","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:01Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling HTTPRoute","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"4f277448-0bd8-45b1-a102-80cbd647efc9","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"no public URL discovered","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"4f277448-0bd8-45b1-a102-80cbd647efc9","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Istio Destination Rules","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"4f277448-0bd8-45b1-a102-80cbd647efc9","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Gateway readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"4f277448-0bd8-45b1-a102-80cbd647efc9","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","gateway":"openshift-ingress/e2e-shared-b-9293de","ready":true}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter.evaluateGatewayConditions","msg":"All referenced Gateways are ready","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"4f277448-0bd8-45b1-a102-80cbd647efc9","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"HTTPRoute readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"4f277448-0bd8-45b1-a102-80cbd647efc9","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","route":"ai-tenant-e2e-shared-b-9293de/rate-test-model-9293de-kserve-route","ready":true}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling Monitoring Resources for LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"4f277448-0bd8-45b1-a102-80cbd647efc9","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService metrics reader RBAC for namespace","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"4f277448-0bd8-45b1-a102-80cbd647efc9","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService engine monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"4f277448-0bd8-45b1-a102-80cbd647efc9","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService scheduler monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"4f277448-0bd8-45b1-a102-80cbd647efc9","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-2-simulated","reconcileID":"bf2b2471-0ad9-4cda-befe-fa7565459f07","Namespace":"llm","Name":"e2e-distinct-2-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-2-simulated","reconcileID":"bf2b2471-0ad9-4cda-befe-fa7565459f07","Namespace":"llm","Name":"e2e-distinct-2-simulated","name":"e2e-distinct-2-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-2-simulated","reconcileID":"bf2b2471-0ad9-4cda-befe-fa7565459f07","Namespace":"llm","Name":"e2e-distinct-2-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-2-simulated","reconcileID":"bf2b2471-0ad9-4cda-befe-fa7565459f07","Namespace":"llm","Name":"e2e-distinct-2-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-2-simulated","reconcileID":"bf2b2471-0ad9-4cda-befe-fa7565459f07","Namespace":"llm","Name":"e2e-distinct-2-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-2-simulated","reconcileID":"bf2b2471-0ad9-4cda-befe-fa7565459f07","Namespace":"llm","Name":"e2e-distinct-2-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Router","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-2-simulated","reconcileID":"bf2b2471-0ad9-4cda-befe-fa7565459f07","Namespace":"llm","Name":"e2e-distinct-2-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Scheduler","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-2-simulated","reconcileID":"bf2b2471-0ad9-4cda-befe-fa7565459f07","Namespace":"llm","Name":"e2e-distinct-2-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling HTTPRoute","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-2-simulated","reconcileID":"bf2b2471-0ad9-4cda-befe-fa7565459f07","Namespace":"llm","Name":"e2e-distinct-2-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"no public URL discovered","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-2-simulated","reconcileID":"bf2b2471-0ad9-4cda-befe-fa7565459f07","Namespace":"llm","Name":"e2e-distinct-2-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Istio Destination Rules","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-2-simulated","reconcileID":"bf2b2471-0ad9-4cda-befe-fa7565459f07","Namespace":"llm","Name":"e2e-distinct-2-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Gateway readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-2-simulated","reconcileID":"bf2b2471-0ad9-4cda-befe-fa7565459f07","Namespace":"llm","Name":"e2e-distinct-2-simulated","gateway":"openshift-ingress/maas-default-gateway","ready":true}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter.evaluateGatewayConditions","msg":"All referenced Gateways are ready","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-2-simulated","reconcileID":"bf2b2471-0ad9-4cda-befe-fa7565459f07","Namespace":"llm","Name":"e2e-distinct-2-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"HTTPRoute readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-2-simulated","reconcileID":"bf2b2471-0ad9-4cda-befe-fa7565459f07","Namespace":"llm","Name":"e2e-distinct-2-simulated","route":"llm/e2e-distinct-2-simulated-kserve-route","ready":true}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling Monitoring Resources for LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-2-simulated","reconcileID":"bf2b2471-0ad9-4cda-befe-fa7565459f07","Namespace":"llm","Name":"e2e-distinct-2-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService metrics reader RBAC for namespace","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-2-simulated","reconcileID":"bf2b2471-0ad9-4cda-befe-fa7565459f07","Namespace":"llm","Name":"e2e-distinct-2-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService engine monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-2-simulated","reconcileID":"bf2b2471-0ad9-4cda-befe-fa7565459f07","Namespace":"llm","Name":"e2e-distinct-2-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService scheduler monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-2-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-2-simulated","reconcileID":"bf2b2471-0ad9-4cda-befe-fa7565459f07","Namespace":"llm","Name":"e2e-distinct-2-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-simulated","reconcileID":"99228808-ca41-4e81-bab5-75c962773968","Namespace":"llm","Name":"e2e-distinct-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-simulated","reconcileID":"99228808-ca41-4e81-bab5-75c962773968","Namespace":"llm","Name":"e2e-distinct-simulated","name":"e2e-distinct-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-simulated","reconcileID":"99228808-ca41-4e81-bab5-75c962773968","Namespace":"llm","Name":"e2e-distinct-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-simulated","reconcileID":"99228808-ca41-4e81-bab5-75c962773968","Namespace":"llm","Name":"e2e-distinct-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-simulated","reconcileID":"99228808-ca41-4e81-bab5-75c962773968","Namespace":"llm","Name":"e2e-distinct-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-simulated","reconcileID":"99228808-ca41-4e81-bab5-75c962773968","Namespace":"llm","Name":"e2e-distinct-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Router","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-simulated","reconcileID":"99228808-ca41-4e81-bab5-75c962773968","Namespace":"llm","Name":"e2e-distinct-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Scheduler","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-simulated","reconcileID":"99228808-ca41-4e81-bab5-75c962773968","Namespace":"llm","Name":"e2e-distinct-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling HTTPRoute","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-simulated","reconcileID":"99228808-ca41-4e81-bab5-75c962773968","Namespace":"llm","Name":"e2e-distinct-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"no public URL discovered","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-simulated","reconcileID":"99228808-ca41-4e81-bab5-75c962773968","Namespace":"llm","Name":"e2e-distinct-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Istio Destination Rules","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-simulated","reconcileID":"99228808-ca41-4e81-bab5-75c962773968","Namespace":"llm","Name":"e2e-distinct-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Gateway readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-simulated","reconcileID":"99228808-ca41-4e81-bab5-75c962773968","Namespace":"llm","Name":"e2e-distinct-simulated","gateway":"openshift-ingress/maas-default-gateway","ready":true}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter.evaluateGatewayConditions","msg":"All referenced Gateways are ready","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-simulated","reconcileID":"99228808-ca41-4e81-bab5-75c962773968","Namespace":"llm","Name":"e2e-distinct-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"HTTPRoute readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-simulated","reconcileID":"99228808-ca41-4e81-bab5-75c962773968","Namespace":"llm","Name":"e2e-distinct-simulated","route":"llm/e2e-distinct-simulated-kserve-route","ready":true}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling Monitoring Resources for LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-simulated","reconcileID":"99228808-ca41-4e81-bab5-75c962773968","Namespace":"llm","Name":"e2e-distinct-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService metrics reader RBAC for namespace","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-simulated","reconcileID":"99228808-ca41-4e81-bab5-75c962773968","Namespace":"llm","Name":"e2e-distinct-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService engine monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-simulated","reconcileID":"99228808-ca41-4e81-bab5-75c962773968","Namespace":"llm","Name":"e2e-distinct-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService scheduler monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"e2e-distinct-simulated","namespace":"llm"},"namespace":"llm","name":"e2e-distinct-simulated","reconcileID":"99228808-ca41-4e81-bab5-75c962773968","Namespace":"llm","Name":"e2e-distinct-simulated"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"2e40dc62-17a7-40ad-8c7b-cd89e329cadf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-shared-a-f22821"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"2e40dc62-17a7-40ad-8c7b-cd89e329cadf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"2e40dc62-17a7-40ad-8c7b-cd89e329cadf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"2e40dc62-17a7-40ad-8c7b-cd89e329cadf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"2e40dc62-17a7-40ad-8c7b-cd89e329cadf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"2e40dc62-17a7-40ad-8c7b-cd89e329cadf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Router","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"2e40dc62-17a7-40ad-8c7b-cd89e329cadf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Scheduler","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"2e40dc62-17a7-40ad-8c7b-cd89e329cadf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:02Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling HTTPRoute","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"2e40dc62-17a7-40ad-8c7b-cd89e329cadf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"no public URL discovered","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"2e40dc62-17a7-40ad-8c7b-cd89e329cadf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Istio Destination Rules","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"2e40dc62-17a7-40ad-8c7b-cd89e329cadf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Gateway readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"2e40dc62-17a7-40ad-8c7b-cd89e329cadf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","gateway":"openshift-ingress/e2e-shared-a-f22821","ready":true}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileRouter.evaluateGatewayConditions","msg":"All referenced Gateways are ready","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"2e40dc62-17a7-40ad-8c7b-cd89e329cadf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"HTTPRoute readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"2e40dc62-17a7-40ad-8c7b-cd89e329cadf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","route":"ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve-route","ready":true}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling Monitoring Resources for LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"2e40dc62-17a7-40ad-8c7b-cd89e329cadf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService metrics reader RBAC for namespace","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"2e40dc62-17a7-40ad-8c7b-cd89e329cadf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService engine monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"2e40dc62-17a7-40ad-8c7b-cd89e329cadf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService scheduler monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"2e40dc62-17a7-40ad-8c7b-cd89e329cadf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"97ea95c3-7250-4ddc-abbb-7214ce5bd569","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"97ea95c3-7250-4ddc-abbb-7214ce5bd569","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"97ea95c3-7250-4ddc-abbb-7214ce5bd569","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"97ea95c3-7250-4ddc-abbb-7214ce5bd569","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"97ea95c3-7250-4ddc-abbb-7214ce5bd569","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"97ea95c3-7250-4ddc-abbb-7214ce5bd569","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Router","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"97ea95c3-7250-4ddc-abbb-7214ce5bd569","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Scheduler","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"97ea95c3-7250-4ddc-abbb-7214ce5bd569","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling HTTPRoute","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"97ea95c3-7250-4ddc-abbb-7214ce5bd569","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"no public URL discovered","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"97ea95c3-7250-4ddc-abbb-7214ce5bd569","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Istio Destination Rules","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"97ea95c3-7250-4ddc-abbb-7214ce5bd569","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Gateway readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"97ea95c3-7250-4ddc-abbb-7214ce5bd569","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","gateway":"openshift-ingress/e2e-shared-b-9293de","ready":true}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileRouter.evaluateGatewayConditions","msg":"All referenced Gateways are ready","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"97ea95c3-7250-4ddc-abbb-7214ce5bd569","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"HTTPRoute readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"97ea95c3-7250-4ddc-abbb-7214ce5bd569","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","route":"ai-tenant-e2e-shared-b-9293de/rate-test-model-9293de-kserve-route","ready":true}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling Monitoring Resources for LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"97ea95c3-7250-4ddc-abbb-7214ce5bd569","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService metrics reader RBAC for namespace","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"97ea95c3-7250-4ddc-abbb-7214ce5bd569","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService engine monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"97ea95c3-7250-4ddc-abbb-7214ce5bd569","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService scheduler monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"97ea95c3-7250-4ddc-abbb-7214ce5bd569","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"9bb9d97d-743b-495d-a3f5-2e2cf0038213","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-shared-a-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"9bb9d97d-743b-495d-a3f5-2e2cf0038213","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"9bb9d97d-743b-495d-a3f5-2e2cf0038213","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"9bb9d97d-743b-495d-a3f5-2e2cf0038213","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"9bb9d97d-743b-495d-a3f5-2e2cf0038213","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"9bb9d97d-743b-495d-a3f5-2e2cf0038213","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Router","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"9bb9d97d-743b-495d-a3f5-2e2cf0038213","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Scheduler","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"9bb9d97d-743b-495d-a3f5-2e2cf0038213","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling HTTPRoute","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"9bb9d97d-743b-495d-a3f5-2e2cf0038213","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"no public URL discovered","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"9bb9d97d-743b-495d-a3f5-2e2cf0038213","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Istio Destination Rules","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"9bb9d97d-743b-495d-a3f5-2e2cf0038213","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Gateway readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"9bb9d97d-743b-495d-a3f5-2e2cf0038213","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","gateway":"openshift-ingress/e2e-shared-a-f22821","ready":true}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileRouter.evaluateGatewayConditions","msg":"All referenced Gateways are ready","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"9bb9d97d-743b-495d-a3f5-2e2cf0038213","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"HTTPRoute readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"9bb9d97d-743b-495d-a3f5-2e2cf0038213","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","route":"ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve-route","ready":true}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling Monitoring Resources for LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"9bb9d97d-743b-495d-a3f5-2e2cf0038213","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService metrics reader RBAC for namespace","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"9bb9d97d-743b-495d-a3f5-2e2cf0038213","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService engine monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"9bb9d97d-743b-495d-a3f5-2e2cf0038213","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:03Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService scheduler monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"9bb9d97d-743b-495d-a3f5-2e2cf0038213","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:31Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"b1ea207d-e806-4a83-b660-92ce1901e33c","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:31Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-shared-a-f22821"}
{"level":"info","ts":"2026-07-06T19:51:31Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:31Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:31Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:31Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:31Z","msg":"Validating LLMInferenceService v1alpha2","name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:51:31Z","msg":"Validating LLMInferenceService v1alpha2","name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:51:31Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:31Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:31Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"b1ea207d-e806-4a83-b660-92ce1901e33c","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"}
{"level":"info","ts":"2026-07-06T19:51:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"b1ea207d-e806-4a83-b660-92ce1901e33c","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"b1ea207d-e806-4a83-b660-92ce1901e33c","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"b1ea207d-e806-4a83-b660-92ce1901e33c","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:64\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"b1ea207d-e806-4a83-b660-92ce1901e33c","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:68\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"b1ea207d-e806-4a83-b660-92ce1901e33c","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"b1ea207d-e806-4a83-b660-92ce1901e33c","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:374\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:44\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"b1ea207d-e806-4a83-b660-92ce1901e33c","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:395\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:47\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"b1ea207d-e806-4a83-b660-92ce1901e33c","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"b1ea207d-e806-4a83-b660-92ce1901e33c","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:424\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:336\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:45\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:85\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Router","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"b1ea207d-e806-4a83-b660-92ce1901e33c","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Scheduler","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"b1ea207d-e806-4a83-b660-92ce1901e33c","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"b1ea207d-e806-4a83-b660-92ce1901e33c","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSchedulerServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/scheduler.go:666\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSchedulerServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/scheduler.go:137\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileScheduler\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/scheduler.go:69\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileRouter\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/router.go:92\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:271\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling HTTPRoute","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"b1ea207d-e806-4a83-b660-92ce1901e33c","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService","msg":"Failed to reconcile LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"b1ea207d-e806-4a83-b660-92ce1901e33c","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"failed to reconcile networking: failed to reconcile HTTP routes: failed to reconcile HTTPRoute ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve-route: failed to create v1.HTTPRoute ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve-route: httproutes.gateway.networking.k8s.io \"rate-test-model-f22821-kserve-route\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-f22821 because it is being terminated","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:232\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:51:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:32Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"}
{"level":"error","ts":"2026-07-06T19:51:32Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"b1ea207d-e806-4a83-b660-92ce1901e33c","error":"failed to reconcile networking: failed to reconcile HTTP routes: failed to reconcile HTTPRoute ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve-route: failed to create v1.HTTPRoute ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve-route: httproutes.gateway.networking.k8s.io \"rate-test-model-f22821-kserve-route\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-f22821 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"393439d5-1ba5-47fe-8335-0346e67c8572","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-shared-a-f22821"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"393439d5-1ba5-47fe-8335-0346e67c8572","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"393439d5-1ba5-47fe-8335-0346e67c8572","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"393439d5-1ba5-47fe-8335-0346e67c8572","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"393439d5-1ba5-47fe-8335-0346e67c8572","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:64\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"393439d5-1ba5-47fe-8335-0346e67c8572","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:68\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"393439d5-1ba5-47fe-8335-0346e67c8572","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"393439d5-1ba5-47fe-8335-0346e67c8572","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:374\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:44\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"393439d5-1ba5-47fe-8335-0346e67c8572","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:395\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:47\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"393439d5-1ba5-47fe-8335-0346e67c8572","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"393439d5-1ba5-47fe-8335-0346e67c8572","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:424\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:336\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:45\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:85\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Router","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"393439d5-1ba5-47fe-8335-0346e67c8572","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Scheduler","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"393439d5-1ba5-47fe-8335-0346e67c8572","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"393439d5-1ba5-47fe-8335-0346e67c8572","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSchedulerServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/scheduler.go:666\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSchedulerServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/scheduler.go:137\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileScheduler\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/scheduler.go:69\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileRouter\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/router.go:92\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:271\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling HTTPRoute","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"393439d5-1ba5-47fe-8335-0346e67c8572","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService","msg":"Failed to reconcile LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"393439d5-1ba5-47fe-8335-0346e67c8572","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"failed to reconcile networking: failed to reconcile HTTP routes: failed to reconcile HTTPRoute ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve-route: failed to create v1.HTTPRoute ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve-route: httproutes.gateway.networking.k8s.io \"rate-test-model-f22821-kserve-route\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-f22821 because it is being terminated","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:232\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:32Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"393439d5-1ba5-47fe-8335-0346e67c8572","error":"failed to reconcile networking: failed to reconcile HTTP routes: failed to reconcile HTTPRoute ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve-route: failed to create v1.HTTPRoute ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve-route: httproutes.gateway.networking.k8s.io \"rate-test-model-f22821-kserve-route\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-f22821 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"c37fb9d9-8775-4814-b9cc-0cdc530ef6bf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-shared-a-f22821"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"c37fb9d9-8775-4814-b9cc-0cdc530ef6bf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"c37fb9d9-8775-4814-b9cc-0cdc530ef6bf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"c37fb9d9-8775-4814-b9cc-0cdc530ef6bf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"c37fb9d9-8775-4814-b9cc-0cdc530ef6bf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:64\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"c37fb9d9-8775-4814-b9cc-0cdc530ef6bf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:68\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"c37fb9d9-8775-4814-b9cc-0cdc530ef6bf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"c37fb9d9-8775-4814-b9cc-0cdc530ef6bf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:374\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:44\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"c37fb9d9-8775-4814-b9cc-0cdc530ef6bf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:395\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:47\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"c37fb9d9-8775-4814-b9cc-0cdc530ef6bf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"c37fb9d9-8775-4814-b9cc-0cdc530ef6bf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:424\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:336\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:45\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:85\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Router","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"c37fb9d9-8775-4814-b9cc-0cdc530ef6bf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Scheduler","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"c37fb9d9-8775-4814-b9cc-0cdc530ef6bf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"c37fb9d9-8775-4814-b9cc-0cdc530ef6bf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSchedulerServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/scheduler.go:666\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSchedulerServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/scheduler.go:137\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileScheduler\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/scheduler.go:69\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileRouter\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/router.go:92\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:271\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling HTTPRoute","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"c37fb9d9-8775-4814-b9cc-0cdc530ef6bf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService","msg":"Failed to reconcile LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"c37fb9d9-8775-4814-b9cc-0cdc530ef6bf","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"failed to reconcile networking: failed to reconcile HTTP routes: failed to reconcile HTTPRoute ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve-route: failed to create v1.HTTPRoute ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve-route: httproutes.gateway.networking.k8s.io \"rate-test-model-f22821-kserve-route\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-f22821 because it is being terminated","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:232\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:32Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"c37fb9d9-8775-4814-b9cc-0cdc530ef6bf","error":"failed to reconcile networking: failed to reconcile HTTP routes: failed to reconcile HTTPRoute ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve-route: failed to create v1.HTTPRoute ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve-route: httproutes.gateway.networking.k8s.io \"rate-test-model-f22821-kserve-route\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-f22821 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"fa95e319-b0ef-4c9f-97d3-135f76e3962d","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-shared-a-f22821"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"fa95e319-b0ef-4c9f-97d3-135f76e3962d","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"fa95e319-b0ef-4c9f-97d3-135f76e3962d","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"fa95e319-b0ef-4c9f-97d3-135f76e3962d","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"fa95e319-b0ef-4c9f-97d3-135f76e3962d","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:64\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"fa95e319-b0ef-4c9f-97d3-135f76e3962d","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:68\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"fa95e319-b0ef-4c9f-97d3-135f76e3962d","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"fa95e319-b0ef-4c9f-97d3-135f76e3962d","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:374\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:44\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"fa95e319-b0ef-4c9f-97d3-135f76e3962d","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:395\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:47\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"fa95e319-b0ef-4c9f-97d3-135f76e3962d","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:32Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"fa95e319-b0ef-4c9f-97d3-135f76e3962d","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:424\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:336\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:45\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:85\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService","msg":"Failed to reconcile LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"fa95e319-b0ef-4c9f-97d3-135f76e3962d","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"failed to reconcile workload: failed to reconcile workload service: failed to create v1.Service ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve-workload-svc: services \"rate-test-model-f22821-kserve-workload-svc\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-f22821 because it is being terminated","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:232\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:33Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"fa95e319-b0ef-4c9f-97d3-135f76e3962d","error":"failed to reconcile workload: failed to reconcile workload service: failed to create v1.Service ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve-workload-svc: services \"rate-test-model-f22821-kserve-workload-svc\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-f22821 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"544c3182-2e5f-4a0b-8b18-b28448b6d211","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-shared-a-f22821"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"544c3182-2e5f-4a0b-8b18-b28448b6d211","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"544c3182-2e5f-4a0b-8b18-b28448b6d211","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"544c3182-2e5f-4a0b-8b18-b28448b6d211","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"544c3182-2e5f-4a0b-8b18-b28448b6d211","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:64\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"544c3182-2e5f-4a0b-8b18-b28448b6d211","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:68\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"544c3182-2e5f-4a0b-8b18-b28448b6d211","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"544c3182-2e5f-4a0b-8b18-b28448b6d211","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:374\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:44\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"544c3182-2e5f-4a0b-8b18-b28448b6d211","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:395\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:47\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"544c3182-2e5f-4a0b-8b18-b28448b6d211","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"544c3182-2e5f-4a0b-8b18-b28448b6d211","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:424\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:336\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:45\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:85\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService","msg":"Failed to reconcile LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"544c3182-2e5f-4a0b-8b18-b28448b6d211","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"failed to reconcile workload: failed to reconcile workload service: failed to create v1.Service ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve-workload-svc: services \"rate-test-model-f22821-kserve-workload-svc\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-f22821 because it is being terminated","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:232\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:33Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"544c3182-2e5f-4a0b-8b18-b28448b6d211","error":"failed to reconcile workload: failed to reconcile workload service: failed to create v1.Service ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve-workload-svc: services \"rate-test-model-f22821-kserve-workload-svc\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-f22821 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"8e6dc64b-524d-4a73-8e6f-0a94874325b4","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-shared-a-f22821"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"8e6dc64b-524d-4a73-8e6f-0a94874325b4","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"8e6dc64b-524d-4a73-8e6f-0a94874325b4","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"8e6dc64b-524d-4a73-8e6f-0a94874325b4","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"8e6dc64b-524d-4a73-8e6f-0a94874325b4","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:64\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"8e6dc64b-524d-4a73-8e6f-0a94874325b4","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:68\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"8e6dc64b-524d-4a73-8e6f-0a94874325b4","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"8e6dc64b-524d-4a73-8e6f-0a94874325b4","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:374\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:44\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"8e6dc64b-524d-4a73-8e6f-0a94874325b4","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:395\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:47\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"8e6dc64b-524d-4a73-8e6f-0a94874325b4","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"8e6dc64b-524d-4a73-8e6f-0a94874325b4","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:424\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:336\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:45\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:85\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService","msg":"Failed to reconcile LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"8e6dc64b-524d-4a73-8e6f-0a94874325b4","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"failed to reconcile workload: failed to reconcile workload service: failed to create v1.Service ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve-workload-svc: services \"rate-test-model-f22821-kserve-workload-svc\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-f22821 because it is being terminated","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:232\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:33Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"8e6dc64b-524d-4a73-8e6f-0a94874325b4","error":"failed to reconcile workload: failed to reconcile workload service: failed to create v1.Service ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve-workload-svc: services \"rate-test-model-f22821-kserve-workload-svc\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-f22821 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"69d51900-4edc-4cd6-a4c2-d16cde2e9cea","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-shared-a-f22821"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"69d51900-4edc-4cd6-a4c2-d16cde2e9cea","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"69d51900-4edc-4cd6-a4c2-d16cde2e9cea","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"69d51900-4edc-4cd6-a4c2-d16cde2e9cea","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"69d51900-4edc-4cd6-a4c2-d16cde2e9cea","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:64\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"69d51900-4edc-4cd6-a4c2-d16cde2e9cea","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:68\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"69d51900-4edc-4cd6-a4c2-d16cde2e9cea","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"69d51900-4edc-4cd6-a4c2-d16cde2e9cea","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:374\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:44\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"69d51900-4edc-4cd6-a4c2-d16cde2e9cea","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:395\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:47\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"69d51900-4edc-4cd6-a4c2-d16cde2e9cea","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"69d51900-4edc-4cd6-a4c2-d16cde2e9cea","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:424\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:336\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:45\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:85\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService","msg":"Failed to reconcile LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"69d51900-4edc-4cd6-a4c2-d16cde2e9cea","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"failed to reconcile workload: failed to reconcile workload service: failed to create v1.Service ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve-workload-svc: services \"rate-test-model-f22821-kserve-workload-svc\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-f22821 because it is being terminated","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:232\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:33Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"69d51900-4edc-4cd6-a4c2-d16cde2e9cea","error":"failed to reconcile workload: failed to reconcile workload service: failed to create v1.Service ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve-workload-svc: services \"rate-test-model-f22821-kserve-workload-svc\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-f22821 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"046e67b7-5c06-4b1f-b900-983310aba1d6","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-shared-a-f22821"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"046e67b7-5c06-4b1f-b900-983310aba1d6","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"046e67b7-5c06-4b1f-b900-983310aba1d6","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"046e67b7-5c06-4b1f-b900-983310aba1d6","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"046e67b7-5c06-4b1f-b900-983310aba1d6","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:64\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"046e67b7-5c06-4b1f-b900-983310aba1d6","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:68\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"046e67b7-5c06-4b1f-b900-983310aba1d6","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"046e67b7-5c06-4b1f-b900-983310aba1d6","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:374\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:44\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"046e67b7-5c06-4b1f-b900-983310aba1d6","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:395\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:47\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"046e67b7-5c06-4b1f-b900-983310aba1d6","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"046e67b7-5c06-4b1f-b900-983310aba1d6","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:424\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:336\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:45\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:85\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService","msg":"Failed to reconcile LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"046e67b7-5c06-4b1f-b900-983310aba1d6","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve: deployments.apps \"rate-test-model-f22821-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-f22821 because it is being terminated","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:232\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService","msg":"Failed to update LLMInferenceService status","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"046e67b7-5c06-4b1f-b900-983310aba1d6","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","LLMInferenceService":"rate-test-model-f22821","error":"Operation cannot be fulfilled on llminferenceservices.serving.kserve.io \"rate-test-model-f22821\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).updateStatus.func1\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:318\nk8s.io/client-go/util/retry.OnError.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.34.5/util/retry/util.go:51\nk8s.io/apimachinery/pkg/util/wait.runConditionWithCrashProtection\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.34.5/pkg/util/wait/wait.go:150\nk8s.io/apimachinery/pkg/util/wait.ExponentialBackoff\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.34.5/pkg/util/wait/backoff.go:477\nk8s.io/client-go/util/retry.OnError\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.34.5/util/retry/util.go:50\nk8s.io/client-go/util/retry.RetryOnConflict\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.34.5/util/retry/util.go:104\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).updateStatus\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:303\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:236\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:51:33Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"046e67b7-5c06-4b1f-b900-983310aba1d6","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-shared-a-f22821/rate-test-model-f22821-kserve: deployments.apps \"rate-test-model-f22821-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-a-f22821 because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"d2d49f2d-988e-4e7f-920a-184fd31fefdd","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService","msg":"Marked for deletion, finalizing resources","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"d2d49f2d-988e-4e7f-920a-184fd31fefdd","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"d2d49f2d-988e-4e7f-920a-184fd31fefdd","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSchedulerServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/scheduler.go:666\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSchedulerServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/scheduler.go:137\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).finalize\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:286\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:201\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.cleanupMonitoring","msg":"Cleaning up monitoring resources - last LLMInferenceService in namespace","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"d2d49f2d-988e-4e7f-920a-184fd31fefdd","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"a6092499-3c9d-45a3-bebf-4f17e1161733","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService","msg":"Marked for deletion, finalizing resources","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"a6092499-3c9d-45a3-bebf-4f17e1161733","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"error","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"a6092499-3c9d-45a3-bebf-4f17e1161733","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSchedulerServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/scheduler.go:666\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSchedulerServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/scheduler.go:137\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).finalize\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:286\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:201\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService.cleanupMonitoring","msg":"Cleaning up monitoring resources - last LLMInferenceService in namespace","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"a6092499-3c9d-45a3-bebf-4f17e1161733","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"}
{"level":"error","ts":"2026-07-06T19:51:33Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"a6092499-3c9d-45a3-bebf-4f17e1161733","error":"llminferenceservices.serving.kserve.io \"rate-test-model-f22821\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"957612ac-fa82-4447-9201-f6d08f3f0c19","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:33Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-f22821","namespace":"ai-tenant-e2e-shared-a-f22821"},"namespace":"ai-tenant-e2e-shared-a-f22821","name":"rate-test-model-f22821","reconcileID":"3f04d4d5-1bfe-4f3f-b45f-53e96d0d394a","Namespace":"ai-tenant-e2e-shared-a-f22821","Name":"rate-test-model-f22821"}
{"level":"info","ts":"2026-07-06T19:51:42Z","msg":"Validating LLMInferenceService v1alpha2","name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:51:42Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:42Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:43Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:43Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:43Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:43Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:44Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:44Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:44Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:44Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:44Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:44Z","msg":"Validating LLMInferenceService v1alpha2","name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:51:44Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:51:45Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"8b358ec2-40cd-4423-91d4-69e631db3dc2","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:45Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"8b358ec2-40cd-4423-91d4-69e631db3dc2","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:51:45Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"8b358ec2-40cd-4423-91d4-69e631db3dc2","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:45Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"8b358ec2-40cd-4423-91d4-69e631db3dc2","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"8b358ec2-40cd-4423-91d4-69e631db3dc2","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:45Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"8b358ec2-40cd-4423-91d4-69e631db3dc2","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Router","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"8b358ec2-40cd-4423-91d4-69e631db3dc2","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:45Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Scheduler","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"8b358ec2-40cd-4423-91d4-69e631db3dc2","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:45Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling HTTPRoute","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"8b358ec2-40cd-4423-91d4-69e631db3dc2","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"no public URL discovered","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"8b358ec2-40cd-4423-91d4-69e631db3dc2","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:45Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Istio Destination Rules","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"8b358ec2-40cd-4423-91d4-69e631db3dc2","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:45Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:51:45Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Gateway readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"8b358ec2-40cd-4423-91d4-69e631db3dc2","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","gateway":"openshift-ingress/e2e-shared-b-9293de","ready":true}
{"level":"info","ts":"2026-07-06T19:51:45Z","logger":"LLMInferenceService.reconcile.reconcileRouter.evaluateGatewayConditions","msg":"All referenced Gateways are ready","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"8b358ec2-40cd-4423-91d4-69e631db3dc2","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:45Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"HTTPRoute readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"8b358ec2-40cd-4423-91d4-69e631db3dc2","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","route":"ai-tenant-e2e-shared-b-9293de/rate-test-model-9293de-kserve-route","ready":true}
{"level":"info","ts":"2026-07-06T19:51:45Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling Monitoring Resources for LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"8b358ec2-40cd-4423-91d4-69e631db3dc2","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:45Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService metrics reader RBAC for namespace","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"8b358ec2-40cd-4423-91d4-69e631db3dc2","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:45Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService engine monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"8b358ec2-40cd-4423-91d4-69e631db3dc2","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:46Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService scheduler monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"8b358ec2-40cd-4423-91d4-69e631db3dc2","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"5f5959bb-3b48-4c44-9738-d00b86bba6cb","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"5f5959bb-3b48-4c44-9738-d00b86bba6cb","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"5f5959bb-3b48-4c44-9738-d00b86bba6cb","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"5f5959bb-3b48-4c44-9738-d00b86bba6cb","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"5f5959bb-3b48-4c44-9738-d00b86bba6cb","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"5f5959bb-3b48-4c44-9738-d00b86bba6cb","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Router","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"5f5959bb-3b48-4c44-9738-d00b86bba6cb","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Scheduler","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"5f5959bb-3b48-4c44-9738-d00b86bba6cb","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling HTTPRoute","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"5f5959bb-3b48-4c44-9738-d00b86bba6cb","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"no public URL discovered","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"5f5959bb-3b48-4c44-9738-d00b86bba6cb","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Istio Destination Rules","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"5f5959bb-3b48-4c44-9738-d00b86bba6cb","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Gateway readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"5f5959bb-3b48-4c44-9738-d00b86bba6cb","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","gateway":"openshift-ingress/e2e-shared-b-9293de","ready":true}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileRouter.evaluateGatewayConditions","msg":"All referenced Gateways are ready","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"5f5959bb-3b48-4c44-9738-d00b86bba6cb","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"HTTPRoute readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"5f5959bb-3b48-4c44-9738-d00b86bba6cb","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","route":"ai-tenant-e2e-shared-b-9293de/rate-test-model-9293de-kserve-route","ready":true}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling Monitoring Resources for LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"5f5959bb-3b48-4c44-9738-d00b86bba6cb","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService metrics reader RBAC for namespace","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"5f5959bb-3b48-4c44-9738-d00b86bba6cb","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService engine monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"5f5959bb-3b48-4c44-9738-d00b86bba6cb","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService scheduler monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"5f5959bb-3b48-4c44-9738-d00b86bba6cb","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1f06caad-a11d-47cd-8081-65e774c8f6da","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1f06caad-a11d-47cd-8081-65e774c8f6da","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1f06caad-a11d-47cd-8081-65e774c8f6da","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1f06caad-a11d-47cd-8081-65e774c8f6da","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1f06caad-a11d-47cd-8081-65e774c8f6da","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1f06caad-a11d-47cd-8081-65e774c8f6da","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Router","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1f06caad-a11d-47cd-8081-65e774c8f6da","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Scheduler","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1f06caad-a11d-47cd-8081-65e774c8f6da","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling HTTPRoute","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1f06caad-a11d-47cd-8081-65e774c8f6da","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"no public URL discovered","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1f06caad-a11d-47cd-8081-65e774c8f6da","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Reconciling Istio Destination Rules","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1f06caad-a11d-47cd-8081-65e774c8f6da","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"Gateway readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1f06caad-a11d-47cd-8081-65e774c8f6da","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","gateway":"openshift-ingress/e2e-shared-b-9293de","ready":true}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileRouter.evaluateGatewayConditions","msg":"All referenced Gateways are ready","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1f06caad-a11d-47cd-8081-65e774c8f6da","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileRouter","msg":"HTTPRoute readiness evaluated","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1f06caad-a11d-47cd-8081-65e774c8f6da","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","route":"ai-tenant-e2e-shared-b-9293de/rate-test-model-9293de-kserve-route","ready":true}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling Monitoring Resources for LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1f06caad-a11d-47cd-8081-65e774c8f6da","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService metrics reader RBAC for namespace","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1f06caad-a11d-47cd-8081-65e774c8f6da","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService engine monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1f06caad-a11d-47cd-8081-65e774c8f6da","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:51:49Z","logger":"LLMInferenceService.reconcile.reconcileMonitoring","msg":"Reconciling LLMInferenceService scheduler monitor","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1f06caad-a11d-47cd-8081-65e774c8f6da","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"369ac7ca-40b7-425e-9d3c-781b6de419d4","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"369ac7ca-40b7-425e-9d3c-781b6de419d4","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"369ac7ca-40b7-425e-9d3c-781b6de419d4","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"369ac7ca-40b7-425e-9d3c-781b6de419d4","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"369ac7ca-40b7-425e-9d3c-781b6de419d4","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"369ac7ca-40b7-425e-9d3c-781b6de419d4","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService","msg":"Failed to reconcile LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"369ac7ca-40b7-425e-9d3c-781b6de419d4","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-shared-b-9293de/rate-test-model-9293de-kserve: deployments.apps \"rate-test-model-9293de-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-9293de because it is being terminated","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:232\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:52:25Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"369ac7ca-40b7-425e-9d3c-781b6de419d4","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-shared-b-9293de/rate-test-model-9293de-kserve: deployments.apps \"rate-test-model-9293de-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-9293de because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b080bb77-630d-452f-a845-491bfdd70579","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b080bb77-630d-452f-a845-491bfdd70579","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b080bb77-630d-452f-a845-491bfdd70579","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b080bb77-630d-452f-a845-491bfdd70579","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b080bb77-630d-452f-a845-491bfdd70579","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:64\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b080bb77-630d-452f-a845-491bfdd70579","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:68\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b080bb77-630d-452f-a845-491bfdd70579","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b080bb77-630d-452f-a845-491bfdd70579","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:374\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:44\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b080bb77-630d-452f-a845-491bfdd70579","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:395\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:47\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b080bb77-630d-452f-a845-491bfdd70579","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b080bb77-630d-452f-a845-491bfdd70579","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:424\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:336\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:45\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:85\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService","msg":"Failed to reconcile LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b080bb77-630d-452f-a845-491bfdd70579","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-shared-b-9293de/rate-test-model-9293de-kserve: deployments.apps \"rate-test-model-9293de-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-9293de because it is being terminated","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:232\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:52:25Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b080bb77-630d-452f-a845-491bfdd70579","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-shared-b-9293de/rate-test-model-9293de-kserve: deployments.apps \"rate-test-model-9293de-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-9293de because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1b0dfae9-4450-4ee8-be13-8217cf8bbab8","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1b0dfae9-4450-4ee8-be13-8217cf8bbab8","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1b0dfae9-4450-4ee8-be13-8217cf8bbab8","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1b0dfae9-4450-4ee8-be13-8217cf8bbab8","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1b0dfae9-4450-4ee8-be13-8217cf8bbab8","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:64\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1b0dfae9-4450-4ee8-be13-8217cf8bbab8","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:68\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1b0dfae9-4450-4ee8-be13-8217cf8bbab8","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1b0dfae9-4450-4ee8-be13-8217cf8bbab8","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:374\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:44\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1b0dfae9-4450-4ee8-be13-8217cf8bbab8","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:395\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:47\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1b0dfae9-4450-4ee8-be13-8217cf8bbab8","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1b0dfae9-4450-4ee8-be13-8217cf8bbab8","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:424\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:336\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:45\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:85\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService","msg":"Failed to reconcile LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1b0dfae9-4450-4ee8-be13-8217cf8bbab8","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-shared-b-9293de/rate-test-model-9293de-kserve: deployments.apps \"rate-test-model-9293de-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-9293de because it is being terminated","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:232\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:52:25Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"1b0dfae9-4450-4ee8-be13-8217cf8bbab8","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-shared-b-9293de/rate-test-model-9293de-kserve: deployments.apps \"rate-test-model-9293de-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-9293de because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"a4a4ee1e-804f-4dd1-bc6a-2c9e2de54b1a","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"a4a4ee1e-804f-4dd1-bc6a-2c9e2de54b1a","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"a4a4ee1e-804f-4dd1-bc6a-2c9e2de54b1a","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"a4a4ee1e-804f-4dd1-bc6a-2c9e2de54b1a","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"a4a4ee1e-804f-4dd1-bc6a-2c9e2de54b1a","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:64\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"a4a4ee1e-804f-4dd1-bc6a-2c9e2de54b1a","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:68\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"a4a4ee1e-804f-4dd1-bc6a-2c9e2de54b1a","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"a4a4ee1e-804f-4dd1-bc6a-2c9e2de54b1a","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:374\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:44\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"a4a4ee1e-804f-4dd1-bc6a-2c9e2de54b1a","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:395\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:47\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"a4a4ee1e-804f-4dd1-bc6a-2c9e2de54b1a","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"a4a4ee1e-804f-4dd1-bc6a-2c9e2de54b1a","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:424\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:336\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:45\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:85\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService","msg":"Failed to reconcile LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"a4a4ee1e-804f-4dd1-bc6a-2c9e2de54b1a","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-shared-b-9293de/rate-test-model-9293de-kserve: deployments.apps \"rate-test-model-9293de-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-9293de because it is being terminated","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:232\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:52:25Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"a4a4ee1e-804f-4dd1-bc6a-2c9e2de54b1a","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-shared-b-9293de/rate-test-model-9293de-kserve: deployments.apps \"rate-test-model-9293de-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-9293de because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b77edfe8-6e58-4fd6-ad3a-3e544b53dbd0","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b77edfe8-6e58-4fd6-ad3a-3e544b53dbd0","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b77edfe8-6e58-4fd6-ad3a-3e544b53dbd0","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b77edfe8-6e58-4fd6-ad3a-3e544b53dbd0","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b77edfe8-6e58-4fd6-ad3a-3e544b53dbd0","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:64\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b77edfe8-6e58-4fd6-ad3a-3e544b53dbd0","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:68\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b77edfe8-6e58-4fd6-ad3a-3e544b53dbd0","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b77edfe8-6e58-4fd6-ad3a-3e544b53dbd0","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:374\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:44\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b77edfe8-6e58-4fd6-ad3a-3e544b53dbd0","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:395\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:47\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b77edfe8-6e58-4fd6-ad3a-3e544b53dbd0","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b77edfe8-6e58-4fd6-ad3a-3e544b53dbd0","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:424\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:336\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:45\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:85\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService","msg":"Failed to reconcile LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b77edfe8-6e58-4fd6-ad3a-3e544b53dbd0","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-shared-b-9293de/rate-test-model-9293de-kserve: deployments.apps \"rate-test-model-9293de-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-9293de because it is being terminated","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:232\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:52:25Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"b77edfe8-6e58-4fd6-ad3a-3e544b53dbd0","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-shared-b-9293de/rate-test-model-9293de-kserve: deployments.apps \"rate-test-model-9293de-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-9293de because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"d91e55ce-a1c7-47c3-8b74-b8e50714e7b4","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"CaBundleConfigMapReconciler","msg":"Reconciling CaBundleConfigMap","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile","msg":"Validating LLMInferenceService v1alpha2","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"d91e55ce-a1c7-47c3-8b74-b8e50714e7b4","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling Workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"d91e55ce-a1c7-47c3-8b74-b8e50714e7b4","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling self-signed certificates secret","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"d91e55ce-a1c7-47c3-8b74-b8e50714e7b4","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"d91e55ce-a1c7-47c3-8b74-b8e50714e7b4","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:64\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"d91e55ce-a1c7-47c3-8b74-b8e50714e7b4","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:68\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeSCCRoleBinding\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:53\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkloadPlatformPermissions\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_permissions_odh.go:46\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:70\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling multi-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"d91e55ce-a1c7-47c3-8b74-b8e50714e7b4","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"d91e55ce-a1c7-47c3-8b74-b8e50714e7b4","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:481\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:374\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:44\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"d91e55ce-a1c7-47c3-8b74-b8e50714e7b4","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:526\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodePrefillServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:395\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileMultiNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_multi_node.go:47\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:79\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Reconciling single-node workload","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"d91e55ce-a1c7-47c3-8b74-b8e50714e7b4","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.reconcile.reconcileWorkload","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"d91e55ce-a1c7-47c3-8b74-b8e50714e7b4","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:424\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeMainServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:336\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSingleNodeWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload_single_node.go:45\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileWorkload\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:85\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:267\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:228\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService","msg":"Failed to reconcile LLMInferenceService","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"d91e55ce-a1c7-47c3-8b74-b8e50714e7b4","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-shared-b-9293de/rate-test-model-9293de-kserve: deployments.apps \"rate-test-model-9293de-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-9293de because it is being terminated","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:232\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"error","ts":"2026-07-06T19:52:25Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"d91e55ce-a1c7-47c3-8b74-b8e50714e7b4","error":"failed to reconcile workload: failed to reconcile single node workload: failed to reconcile main workload: failed to create v1.Deployment ai-tenant-e2e-shared-b-9293de/rate-test-model-9293de-kserve: deployments.apps \"rate-test-model-9293de-kserve\" is forbidden: unable to create new content in namespace ai-tenant-e2e-shared-b-9293de because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"ce7ba797-6879-4d5f-8889-1919a3cfb2e5","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService","msg":"Marked for deletion, finalizing resources","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"ce7ba797-6879-4d5f-8889-1919a3cfb2e5","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"error","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"ce7ba797-6879-4d5f-8889-1919a3cfb2e5","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSchedulerServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/scheduler.go:666\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSchedulerServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/scheduler.go:137\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).finalize\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:286\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:201\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:52:25Z","logger":"LLMInferenceService.cleanupMonitoring","msg":"Cleaning up monitoring resources - last LLMInferenceService in namespace","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"ce7ba797-6879-4d5f-8889-1919a3cfb2e5","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"info","ts":"2026-07-06T19:52:26Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"86964b5e-e70b-4d39-8435-42e95933886d","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:52:26Z","logger":"LLMInferenceService","msg":"Marked for deletion, finalizing resources","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"86964b5e-e70b-4d39-8435-42e95933886d","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"error","ts":"2026-07-06T19:52:26Z","logger":"LLMInferenceService","msg":"Warning: failed to retrieve 'default' service account, continuing ...","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"86964b5e-e70b-4d39-8435-42e95933886d","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","error":"ServiceAccount \"default\" not found","stacktrace":"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).injectSecretsFromDefaultServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/workload.go:190\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).expectedSchedulerServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/scheduler.go:666\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).reconcileSchedulerServiceAccount\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/scheduler.go:137\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).finalize\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:286\ngithub.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc.(*LLMISVCReconciler).Reconcile\n\t/go/src/github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc/controller.go:201\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:116\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:303\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:52:26Z","logger":"LLMInferenceService.cleanupMonitoring","msg":"Cleaning up monitoring resources - last LLMInferenceService in namespace","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"86964b5e-e70b-4d39-8435-42e95933886d","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"}
{"level":"error","ts":"2026-07-06T19:52:26Z","msg":"Reconciler error","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"86964b5e-e70b-4d39-8435-42e95933886d","error":"llminferenceservices.serving.kserve.io \"rate-test-model-9293de\" not found","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224"}
{"level":"info","ts":"2026-07-06T19:52:26Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"25dc8070-e140-42bf-b95e-6fff3cf02eae","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:52:26Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"96e875bf-c11d-4b47-8ab0-77294d555cda","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:52:26Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"dfc470e5-2eaf-4a9b-a3f4-985204c459e6","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:52:26Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"62822330-7b91-44fa-8e45-3392418c8413","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:52:26Z","logger":"LLMInferenceService","msg":"Starting reconciliation","controller":"llminferenceservice","controllerGroup":"serving.kserve.io","controllerKind":"LLMInferenceService","LLMInferenceService":{"name":"rate-test-model-9293de","namespace":"ai-tenant-e2e-shared-b-9293de"},"namespace":"ai-tenant-e2e-shared-b-9293de","name":"rate-test-model-9293de","reconcileID":"bf627e70-035b-428c-ac47-e8ea803bbf9d","Namespace":"ai-tenant-e2e-shared-b-9293de","Name":"rate-test-model-9293de"}
{"level":"info","ts":"2026-07-06T19:52:26Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:52:26Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:52:26Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:52:26Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:52:26Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:52:26Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:52:26Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:52:26Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:52:26Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:52:26Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:52:26Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:52:26Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:52:26Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:52:26Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:52:26Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:52:26Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:52:26Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:52:26Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:52:35Z","msg":"Validating LLMInferenceService v1alpha2","name":"premium-simulated-simulated-premium","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:52:35Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-2-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:52:35Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-trlp-test-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:52:35Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-unconfigured-facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:52:35Z","msg":"Validating LLMInferenceService v1alpha2","name":"facebook-opt-125m-simulated","namespace":"llm"}
{"level":"info","ts":"2026-07-06T19:52:35Z","msg":"Validating LLMInferenceService v1alpha2","name":"e2e-distinct-simulated","namespace":"llm"}