2026-10-30T12:45:14.702101Z info FLAG: --concurrency="0" 2026-10-30T12:45:14.702121Z info FLAG: --domain="openshift-ingress.svc.cluster.local" 2026-10-30T12:45:14.702127Z info FLAG: --help="false" 2026-10-30T12:45:14.702130Z info FLAG: --log_as_json="false" 2026-10-30T12:45:14.702133Z info FLAG: --log_caller="" 2026-10-30T12:45:14.702136Z info FLAG: --log_output_level="default:info" 2026-10-30T12:45:14.702139Z info FLAG: --log_stacktrace_level="default:none" 2026-10-30T12:45:14.702144Z info FLAG: --log_target="[stdout]" 2026-10-30T12:45:14.702148Z info FLAG: --meshConfig="./etc/istio/config/mesh" 2026-10-30T12:45:14.702151Z info FLAG: --outlierLogPath="" 2026-10-30T12:45:14.702154Z info FLAG: --profiling="true" 2026-10-30T12:45:14.702157Z info FLAG: --proxyComponentLogLevel="misc:error" 2026-10-30T12:45:14.702160Z info FLAG: --proxyLogLevel="warning" 2026-10-30T12:45:14.702163Z info FLAG: --serviceCluster="istio-proxy" 2026-10-30T12:45:14.702165Z info FLAG: --stsPort="0" 2026-10-30T12:45:14.702168Z info FLAG: --templateFile="" 2026-10-30T12:45:14.702171Z info FLAG: --tokenManagerPlugin="" 2026-10-30T12:45:14.702174Z info FLAG: --vklog="0" 2026-10-30T12:45:14.702178Z info Version 1.28.10-cebc47d567fd1333707199224c18129ecfe07c05-Clean 2026-10-30T12:45:14.702376Z info Proxy role ips=[10.133.0.41] type=router id=maas-default-gateway-openshift-default-7d9897b54c-9twbb.openshift-ingress domain=openshift-ingress.svc.cluster.local 2026-10-30T12:45:14.702437Z info Apply proxy config from env {"discoveryAddress":"istiod-openshift-gateway.openshift-ingress.svc:15012","proxyHeaders":{"server":{"disabled":true},"envoyDebugHeaders":{"disabled":true},"metadataExchangeHeaders":{"mode":"IN_MESH"}}} 2026-10-30T12:45:14.703929Z info cpu limit detected as 2, setting concurrency 2026-10-30T12:45:14.704444Z info Effective config: binaryPath: /usr/local/bin/envoy concurrency: 2 configPath: ./etc/istio/proxy controlPlaneAuthPolicy: MUTUAL_TLS discoveryAddress: istiod-openshift-gateway.openshift-ingress.svc:15012 drainDuration: 45s proxyAdminPort: 15000 proxyHeaders: envoyDebugHeaders: disabled: true metadataExchangeHeaders: mode: IN_MESH server: disabled: true serviceCluster: istio-proxy statNameLength: 189 statusPort: 15020 terminationDrainDuration: 5s 2026-10-30T12:45:14.704510Z info JWT policy is third-party-jwt 2026-10-30T12:45:14.704537Z info using credential fetcher of JWT type in cluster.local trust domain 2026-10-30T12:45:14.706938Z info Starting default Istio SDS Server 2026-10-30T12:45:14.706979Z info CA Endpoint istiod-openshift-gateway.openshift-ingress.svc:15012, provider Citadel 2026-10-30T12:45:14.707002Z info Using CA istiod-openshift-gateway.openshift-ingress.svc:15012 cert with certs: var/run/secrets/istio/root-cert.pem 2026-10-30T12:45:14.707175Z info Opening status port 15020 2026-10-30T12:45:14.708627Z info xdsproxy Initializing with upstream address "istiod-openshift-gateway.openshift-ingress.svc:15012" and cluster "Kubernetes" 2026-10-30T12:45:14.710397Z info Pilot SAN: [istiod-openshift-gateway.openshift-ingress.svc] 2026-10-30T12:45:14.711554Z info Starting proxy agent 2026-10-30T12:45:14.711631Z info Envoy command: [-c etc/istio/proxy/envoy-rev.json --drain-time-s 45 --drain-strategy immediate --local-address-ip-version v4 --file-flush-interval-msec 1000 --disable-hot-restart --allow-unknown-static-fields -l warning --component-log-level misc:error --skip-deprecated-logs --concurrency 2] 2026-10-30T12:45:14.720503Z info sds Starting SDS grpc server 2026-10-30T12:45:14.720525Z info sds Starting SDS server for workload certificates, will listen on "var/run/secrets/workload-spiffe-uds/socket" 2026-10-30T12:45:14.766849Z info cache generated new workload certificate resourceName=default latency=57.562742ms ttl=23h59m59.233154175s 2026-10-30T12:45:14.766887Z info cache Root cert has changed, start rotating root cert 2026-10-30T12:45:14.766931Z info cache returned workload trust anchor from cache ttl=23h59m59.233069829s 2026-10-30T12:45:14.804911Z info xdsproxy connected to delta upstream XDS server: istiod-openshift-gateway.openshift-ingress.svc:15012 id=1 2026-10-30T12:45:14.942201Z info ads ADS: new connection for node:1 2026-10-30T12:45:14.942278Z info cache returned workload certificate from cache ttl=23h59m59.057726834s 2026-10-30T12:45:14.944326Z info ads ADS: new connection for node:2 2026-10-30T12:45:14.944769Z info cache returned workload trust anchor from cache ttl=23h59m59.055233351s 2026-10-30T12:45:15.244014Z warning envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317 Wasm remote code fetch is unstable and may cause a crash thread=8 2026-10-30T12:45:23.233368Z info Readiness succeeded in 8.534412457s 2026-10-30T12:45:23.233688Z info Envoy proxy is ready 2026-10-30T12:45:44.559918Z warning envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:372 thread=24 2026-10-30T12:45:44.559941Z warning envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:372 thread=24 2026-10-30T12:45:44.559944Z warning envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:372 thread=24 2026-10-30T12:45:44.559948Z warning envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:385 key 'model' missing from lines 1-1 key: model thread=24 [2026-10-30T12:45:44.494Z] "POST /v1/chat/completions HTTP/1.1" 429 - - "-" 122 87 65 - "44.212.242.249,10.132.0.17" "curl/7.76.1" "e9c6a6e7-d847-4c2e-8ea2-3f6a92f5facc" "maas.apps.745e080318698fdddabc.prod.konflux-ocp-ci.dev" "-" outbound|8000||facebook-opt-125m-simulated-kserve-workload-svc.llm.svc.cluster.local - 10.133.0.41:443 10.132.0.17:53314 - v1-model-routing 2026-10-30T12:49:31.428987Z info ads ADS: new connection for node:3 2026-10-30T12:49:31.491021Z info cache adding watcher for file certificate /etc/pki/tls/certs/ca-bundle.crt 2026-10-30T12:49:31.491079Z info cache read certificate from file resource=file-root:system 2026-10-30T12:49:33.767441Z warning envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317 Wasm remote code fetch is unstable and may cause a crash thread=8 [2026-10-30T12:50:37.175Z] "POST /anything/e2e-rt-sw-bf0674-client HTTP/1.1" 200 - via_upstream - "-" 124 1520 162 - "44.212.242.249,10.132.0.17" "python-requests/2.34.2" "91eb241e-dafb-4b12-a1a2-9f84d21c88c9" "httpbingo.org" "66.241.125.232:443" outbound|443||httpbingo.org 10.133.0.41:53826 10.133.0.41:443 10.132.0.17:42934 - llm.external-model-e2e-rt-sw-bf0674-47cf9f30165c.0 [2026-10-30T12:50:42.989Z] "POST /llm/e2e-rt-sw-bf0674/anything/e2e-rt-sw-bf0674-client HTTP/1.1" 503 UH no_healthy_upstream - "-" 124 19 8 - "44.212.242.249,10.132.0.17" "python-requests/2.34.2" "49873d2e-8874-48d3-a299-892b1131455e" "maas.apps.745e080318698fdddabc.prod.konflux-ocp-ci.dev" "-" outbound|443||provider-selection-required-e2e-rt-sw-bf0674.llm.svc.cluster.local - 10.133.0.41:443 10.132.0.17:42952 - llm.external-model-e2e-rt-sw-bf0674-47cf9f30165c.2 [2026-10-30T12:50:53.055Z] "POST /llm/e2e-rt-sw-bf0674/anything/e2e-rt-sw-bf0674-client HTTP/1.1" 503 UH no_healthy_upstream - "-" 124 19 8 - "44.212.242.249,10.132.0.17" "python-requests/2.34.2" "a278a059-6ca2-496c-bed4-89074103e86e" "maas.apps.745e080318698fdddabc.prod.konflux-ocp-ci.dev" "-" outbound|443||provider-selection-required-e2e-rt-sw-bf0674.llm.svc.cluster.local - 10.133.0.41:443 10.132.0.17:38726 - llm.external-model-e2e-rt-sw-bf0674-47cf9f30165c.2 [2026-10-30T12:52:11.768Z] "POST /anything/e2e-rt-eq-3d0efa-client HTTP/1.1" 200 - via_upstream - "-" 124 1522 58 - "44.212.242.249,10.132.0.17" "python-requests/2.34.2" "e59c884b-243a-4f24-8a75-6bb98a71c314" "httpbingo.org" "66.241.125.232:443" outbound|443||httpbingo.org 10.133.0.41:46000 10.133.0.41:443 10.132.0.17:60150 - llm.external-model-e2e-rt-eq-3d0efa-29a95bd61c7c.2 [2026-10-30T12:52:12.731Z] "POST /anything/e2e-rt-eq-3d0efa-client HTTP/1.1" 200 - via_upstream - "-" 124 1519 48 - "44.212.242.249,10.132.0.17" "python-requests/2.34.2" "dda49032-00df-4fed-94ef-23715a935bd6" "httpbingo.org" "66.241.125.232:443" outbound|443||httpbingo.org 10.133.0.41:46006 10.133.0.41:443 10.132.0.17:60158 - llm.external-model-e2e-rt-eq-3d0efa-29a95bd61c7c.2 2026-10-30T12:52:47.584477Z info ads ADS: "" 3 terminated with stream closed 2026-10-30T12:52:47.853050Z warning envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317 Wasm remote code fetch is unstable and may cause a crash thread=8 2026-10-30T12:56:18.811570Z error envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155 wasm log kuadrant-wasm-shim kuadrant_wasm_shim fd5039db79d000f31669de1fde37ea175d3ad05f2b14341d6936fb564cab5553: Task failed: "0" thread=25 2026-10-30T12:56:19.220953Z error envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155 wasm log kuadrant-wasm-shim kuadrant_wasm_shim fd5039db79d000f31669de1fde37ea175d3ad05f2b14341d6936fb564cab5553: Task failed: "0" thread=25 [2026-10-30T12:56:18.708Z] "POST /v1/api-keys HTTP/1.1" 201 - via_upstream - "-" 71 313 103 - "44.212.242.249,10.132.0.17" "python-requests/2.34.2" "e8dce45e-93db-4aef-8acc-a48ee4690449" "maas.apps.745e080318698fdddabc.prod.konflux-ocp-ci.dev" "10.132.0.52:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.133.0.41:40846 10.133.0.41:443 10.132.0.17:45730 - odh-ai-gateway-infra.maas-api-route.3 [2026-10-30T12:56:18.934Z] "POST /maas-api/v1/api-keys HTTP/1.1" 403 - - "-" 77 0 57 - "44.212.242.249,10.132.0.17" "python-requests/2.34.2" "0ee706bb-8e18-4e25-815a-bc43c17c44c9" "maas.apps.745e080318698fdddabc.prod.konflux-ocp-ci.dev" "-" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local - 10.133.0.41:443 10.132.0.17:45740 - odh-ai-gateway-infra.maas-api-route.3 [2026-10-30T12:56:19.126Z] "DELETE /v1/api-keys/72396a31-6acf-4903-b7ce-3f01276c383d HTTP/1.1" 200 - via_upstream - "-" 0 321 95 - "44.212.242.249,10.132.0.17" "python-requests/2.34.2" "ef0fa40a-0b03-4f04-8624-83adce278019" "maas.apps.745e080318698fdddabc.prod.konflux-ocp-ci.dev" "10.132.0.52:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.133.0.41:40846 10.133.0.41:443 10.132.0.17:45744 - odh-ai-gateway-infra.maas-api-route.3 2026-10-30T12:56:19.924377Z error envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155 wasm log kuadrant-wasm-shim kuadrant_wasm_shim fd5039db79d000f31669de1fde37ea175d3ad05f2b14341d6936fb564cab5553: Task failed: "0" thread=25 [2026-10-30T12:56:19.824Z] "DELETE /v1/api-keys/61310782-bbe7-4b31-a375-0b31f7801195 HTTP/1.1" 200 - via_upstream - "-" 0 321 99 - "44.212.242.249,10.132.0.17" "python-requests/2.34.2" "dd85e9f0-81b7-448f-896d-4da02758d742" "maas.apps.745e080318698fdddabc.prod.konflux-ocp-ci.dev" "10.132.0.52:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.133.0.41:40846 10.133.0.41:443 10.132.0.17:45756 - odh-ai-gateway-infra.maas-api-route.3 2026-10-30T12:56:28.964708Z error envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155 wasm log kuadrant-wasm-shim kuadrant_wasm_shim fd5039db79d000f31669de1fde37ea175d3ad05f2b14341d6936fb564cab5553: Task failed: "0" thread=25 [2026-10-30T12:56:28.879Z] "POST /v1/api-keys HTTP/1.1" 201 - via_upstream - "-" 93 316 85 - "44.212.242.249,10.132.0.17" "python-requests/2.34.2" "5573e713-caea-47f5-876b-0d5f61e748a2" "maas.apps.745e080318698fdddabc.prod.konflux-ocp-ci.dev" "10.132.0.52:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.133.0.41:40846 10.133.0.41:443 10.132.0.17:56562 - odh-ai-gateway-infra.maas-api-route.3 2026-10-30T12:57:11.838782Z error envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155 wasm log kuadrant-wasm-shim kuadrant_wasm_shim fd5039db79d000f31669de1fde37ea175d3ad05f2b14341d6936fb564cab5553: Task failed: "0" thread=25 [2026-10-30T12:57:11.828Z] "GET /health HTTP/1.1" 200 - via_upstream - "-" 0 91 10 - "44.212.242.249,10.132.0.17" "python-requests/2.34.2" "7d54e69e-a7d0-4a7f-a9ad-c09a4d20921a" "maas.apps.745e080318698fdddabc.prod.konflux-ocp-ci.dev" "10.132.0.52:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.133.0.41:40846 10.133.0.41:443 10.132.0.17:43028 - odh-ai-gateway-infra.maas-api-route.4 [2026-10-30T12:59:36.009Z] "POST /maas-api/v1/api-keys HTTP/1.1" 403 - - "-" 77 0 57 - "44.212.242.249,10.132.0.17" "python-requests/2.34.2" "cf7858b0-be7a-4255-8a16-eeda87ece5c0" "maas.apps.745e080318698fdddabc.prod.konflux-ocp-ci.dev" "-" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local - 10.133.0.41:443 10.132.0.17:47168 - odh-ai-gateway-infra.maas-api-route.3 2026-10-30T13:04:29.003475Z error envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155 wasm log kuadrant-wasm-shim kuadrant_wasm_shim fd5039db79d000f31669de1fde37ea175d3ad05f2b14341d6936fb564cab5553: Task failed: "0" thread=25 [2026-10-30T13:04:28.973Z] "GET /health HTTP/1.1" 200 - via_upstream - "-" 0 91 30 - "44.212.242.249,10.132.0.17" "python-requests/2.34.2" "90c42759-ade0-4a1b-9e93-d406e36c7d48" "maas.apps.745e080318698fdddabc.prod.konflux-ocp-ci.dev" "10.132.0.52:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.133.0.41:43622 10.133.0.41:443 10.132.0.17:50896 - odh-ai-gateway-infra.maas-api-route.4 2026-10-30T13:04:29.332291Z error envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155 wasm log kuadrant-wasm-shim kuadrant_wasm_shim fd5039db79d000f31669de1fde37ea175d3ad05f2b14341d6936fb564cab5553: Task failed: "0" thread=24 [2026-10-30T13:04:29.254Z] "POST /v1/api-keys HTTP/1.1" 201 - via_upstream - "-" 79 321 78 - "44.212.242.249,10.132.0.17" "python-requests/2.34.2" "36e15216-06e0-4cce-a2ea-09b76ddbb198" "maas.apps.745e080318698fdddabc.prod.konflux-ocp-ci.dev" "10.132.0.52:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.133.0.41:43628 10.133.0.41:443 10.132.0.17:50898 - odh-ai-gateway-infra.maas-api-route.3 2026-10-30T13:04:50.729853Z error envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155 wasm log kuadrant-wasm-shim kuadrant_wasm_shim fd5039db79d000f31669de1fde37ea175d3ad05f2b14341d6936fb564cab5553: Task failed: "0" thread=24 2026-10-30T13:04:50.730017Z error envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155 wasm log kuadrant-wasm-shim kuadrant_wasm_shim fd5039db79d000f31669de1fde37ea175d3ad05f2b14341d6936fb564cab5553: Failed to apply predicates: EvaluationError { expression: Expression { attributes: [Attribute { path: ["request", "path"] }, Attribute { path: ["auth", "identity", "selected_subscription_id"] }, Attribute { path: ["kuadrant", "internal", "tokenratelimit", "reservation", "tokenlimit_tokens_100_per_1m__cec148f9"] }, Attribute { path: ["kuadrant", "internal", "response", "body", "total_tokens"] }], request_body_values: [], response_body_values: [], request_body_groups: [], response_body_groups: [], expression: IdedExpr { id: 27, expr: Call(CallExpr { func_name: "_&&_", target: None, args: [IdedExpr { id: 11, expr: Call(CallExpr { func_name: "_&&_", target: None, args: [IdedExpr { id: 4, expr: Call(CallExpr { func_name: "_==_", target: None, args: [IdedExpr { id: 3, expr: Select(SelectExpr { operand: IdedExpr { id: 2, expr: Select(SelectExpr { operand: IdedExpr { id: 1, expr: Ident("auth") }, field: "identity", test: false }) }, field: "selected_subscription_id", test: false }) }, IdedExpr { id: 5, expr: Literal(String(String("3f98c74fda8361a9"))) }] }) }, IdedExpr { id: 6, expr: Call(CallExpr { func_name: "!_", target: None, args: [IdedExpr { id: 9, expr: Call(CallExpr { func_name: "endsWith", target: Some(IdedExpr { id: 8, expr: Select(SelectExpr { operand: IdedExpr { id: 7, expr: Ident("request") }, field: "path", test: false }) }), args: [IdedExpr { id: 10, expr: Literal(String(String("/v1/models"))) }] }) }] }) }] }) }, IdedExpr { id: 26, expr: Call(CallExpr { func_name: "_||_", target: None, args: [IdedExpr { id: 17, expr: Call(CallExpr { func_name: "_!=_", target: None, args: [IdedExpr { id: 16, expr: Select(SelectExpr { operand: IdedExpr { id: 15, expr: Select(SelectExpr { operand: IdedExpr { id: 14, expr: Select(SelectExpr { operand: IdedExpr { id: 13, expr: Select(SelectExpr { operand: IdedExpr { id: 12, expr: Ident("kuadrant") }, field: "internal", test: false }) }, field: "tokenratelimit", test: false }) }, field: "reservation", test: false }) }, field: "tokenlimit_tokens_100_per_1m__cec148f9", test: false }) }, IdedExpr { id: 18, expr: Literal(Null) }] }) }, IdedExpr { id: 24, expr: Call(CallExpr { func_name: "_!=_", target: None, args: [IdedExpr { id: 23, expr: Select(SelectExpr { operand: IdedExpr { id: 22, expr: Select(SelectExpr { operand: IdedExpr { id: 21, expr: Select(SelectExpr { operand: IdedExpr { id: 20, expr: Select(SelectExpr { operand: IdedExpr { id: 19, expr: Ident("kuadrant") }, field: "internal", test: false }) }, field: "response", test: false }) }, field: "body", test: false }) }, field: "total_tokens", test: false }) }, IdedExpr { id: 25, expr: Literal(Null) }] }) }] }) }] }) }, extended: false, needs_grpc: false }, message: "CelError::Resolve { UndeclaredReference(\"kuadrant\") }" } thread=24 2026-10-30T13:04:50.730167Z warning envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:372 thread=24 2026-10-30T13:04:50.730177Z warning envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:372 thread=24 2026-10-30T13:04:50.730180Z warning envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:372 thread=24 2026-10-30T13:04:50.730184Z warning envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:385 key 'model' missing from lines 1-1 key: model thread=24 [2026-10-30T13:04:50.630Z] "POST /v1/chat/completions HTTP/1.1" 404 - via_upstream - "-" 120 126 99 - "44.212.242.249,10.132.0.17" "python-requests/2.34.2" "5376280b-6f18-4f93-8d4c-7084c1940851" "maas.apps.745e080318698fdddabc.prod.konflux-ocp-ci.dev" "10.134.0.33:8000" outbound|8000||facebook-opt-125m-simulated-kserve-workload-svc.llm.svc.cluster.local 10.133.0.41:43276 10.133.0.41:443 10.132.0.17:60962 - v1-chat-completions-path