2026-11-03T20:11:32.711912Z	info	FLAG: --concurrency="0"
2026-11-03T20:11:32.711934Z	info	FLAG: --domain="openshift-ingress.svc.cluster.local"
2026-11-03T20:11:32.711940Z	info	FLAG: --help="false"
2026-11-03T20:11:32.711943Z	info	FLAG: --log_as_json="false"
2026-11-03T20:11:32.711946Z	info	FLAG: --log_caller=""
2026-11-03T20:11:32.711949Z	info	FLAG: --log_output_level="default:info"
2026-11-03T20:11:32.711953Z	info	FLAG: --log_stacktrace_level="default:none"
2026-11-03T20:11:32.711959Z	info	FLAG: --log_target="[stdout]"
2026-11-03T20:11:32.711963Z	info	FLAG: --meshConfig="./etc/istio/config/mesh"
2026-11-03T20:11:32.711966Z	info	FLAG: --outlierLogPath=""
2026-11-03T20:11:32.711970Z	info	FLAG: --profiling="true"
2026-11-03T20:11:32.711973Z	info	FLAG: --proxyComponentLogLevel="misc:error"
2026-11-03T20:11:32.711976Z	info	FLAG: --proxyLogLevel="warning"
2026-11-03T20:11:32.711980Z	info	FLAG: --serviceCluster="istio-proxy"
2026-11-03T20:11:32.711983Z	info	FLAG: --stsPort="0"
2026-11-03T20:11:32.711986Z	info	FLAG: --templateFile=""
2026-11-03T20:11:32.711989Z	info	FLAG: --tokenManagerPlugin=""
2026-11-03T20:11:32.711992Z	info	FLAG: --vklog="0"
2026-11-03T20:11:32.711997Z	info	Version 1.28.10-cebc47d567fd1333707199224c18129ecfe07c05-Clean
2026-11-03T20:11:32.712197Z	info	Proxy role	ips=[10.134.0.37] type=router id=maas-default-gateway-openshift-default-59f957fdd7-6bhmr.openshift-ingress domain=openshift-ingress.svc.cluster.local
2026-11-03T20:11:32.712262Z	info	Apply proxy config from env {"discoveryAddress":"istiod-openshift-gateway.openshift-ingress.svc:15012","proxyHeaders":{"server":{"disabled":true},"envoyDebugHeaders":{"disabled":true},"metadataExchangeHeaders":{"mode":"IN_MESH"}}}

2026-11-03T20:11:32.713905Z	info	cpu limit detected as 2, setting concurrency
2026-11-03T20:11:32.714222Z	info	Effective config: binaryPath: /usr/local/bin/envoy
concurrency: 2
configPath: ./etc/istio/proxy
controlPlaneAuthPolicy: MUTUAL_TLS
discoveryAddress: istiod-openshift-gateway.openshift-ingress.svc:15012
drainDuration: 45s
proxyAdminPort: 15000
proxyHeaders:
  envoyDebugHeaders:
    disabled: true
  metadataExchangeHeaders:
    mode: IN_MESH
  server:
    disabled: true
serviceCluster: istio-proxy
statNameLength: 189
statusPort: 15020
terminationDrainDuration: 5s

2026-11-03T20:11:32.714237Z	info	JWT policy is third-party-jwt
2026-11-03T20:11:32.714241Z	info	using credential fetcher of JWT type in cluster.local trust domain
2026-11-03T20:11:32.716579Z	info	Starting default Istio SDS Server
2026-11-03T20:11:32.716630Z	info	CA Endpoint istiod-openshift-gateway.openshift-ingress.svc:15012, provider Citadel
2026-11-03T20:11:32.716655Z	info	Using CA istiod-openshift-gateway.openshift-ingress.svc:15012 cert with certs: var/run/secrets/istio/root-cert.pem
2026-11-03T20:11:32.716916Z	info	Opening status port 15020
2026-11-03T20:11:32.717976Z	info	xdsproxy	Initializing with upstream address "istiod-openshift-gateway.openshift-ingress.svc:15012" and cluster "Kubernetes"
2026-11-03T20:11:32.718520Z	info	sds	Starting SDS grpc server
2026-11-03T20:11:32.718557Z	info	sds	Starting SDS server for workload certificates, will listen on "var/run/secrets/workload-spiffe-uds/socket"
2026-11-03T20:11:32.719518Z	info	Pilot SAN: [istiod-openshift-gateway.openshift-ingress.svc]
2026-11-03T20:11:32.720425Z	info	Starting proxy agent
2026-11-03T20:11:32.720455Z	info	Envoy command: [-c etc/istio/proxy/envoy-rev.json --drain-time-s 45 --drain-strategy immediate --local-address-ip-version v4 --file-flush-interval-msec 1000 --disable-hot-restart --allow-unknown-static-fields -l warning --component-log-level misc:error --skip-deprecated-logs --concurrency 2]
2026-11-03T20:11:32.803076Z	info	xdsproxy	connected to delta upstream XDS server: istiod-openshift-gateway.openshift-ingress.svc:15012	id=1
2026-11-03T20:11:32.906769Z	info	cache	generated new workload certificate	resourceName=default latency=188.044301ms ttl=23h59m59.093237701s
2026-11-03T20:11:32.906804Z	info	cache	Root cert has changed, start rotating root cert
2026-11-03T20:11:32.906838Z	info	cache	returned workload trust anchor from cache	ttl=23h59m59.093162548s
2026-11-03T20:11:32.928943Z	info	ads	ADS: new connection for node:1
2026-11-03T20:11:32.929161Z	info	ads	ADS: new connection for node:2
2026-11-03T20:11:32.929444Z	info	cache	returned workload certificate from cache	ttl=23h59m59.07055825s
2026-11-03T20:11:32.929545Z	info	cache	returned workload trust anchor from cache	ttl=23h59m59.070457103s
2026-11-03T20:11:33.017980Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:11:41.441823Z	info	Readiness succeeded in 8.733393334s
2026-11-03T20:11:41.442144Z	info	Envoy proxy is ready
2026-11-03T20:14:17.060845Z	info	ads	ADS: new connection for node:3
2026-11-03T20:14:17.066131Z	info	cache	adding watcher for file certificate /etc/pki/tls/certs/ca-bundle.crt
2026-11-03T20:14:17.066251Z	info	cache	read certificate from file	resource=file-root:system
2026-11-03T20:14:17.691160Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:14:29.450206Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:14:37.077461Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:14:42.628881Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T20:14:42.628915Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T20:14:42.628925Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
2026-11-03T20:14:42.628930Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T20:14:42.629090Z	error	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Task failed: Some("2")	thread=25
[2026-11-03T20:14:42.535Z] "POST /llm/e2e-rt-sw-9dcd2a/anything/e2e-rt-sw-9dcd2a-client HTTP/1.1" 503 UH no_healthy_upstream - "-" 124 19 93 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "b2813c58-ee0f-4850-9e0c-94566256406f" "maas.apps.96089cf8aa8b05683edc.prod.konflux-ocp-ci.dev" "-" outbound|443||provider-selection-required-e2e-rt-sw-9dcd2a.llm.svc.cluster.local - 10.134.0.37:443 10.132.0.13:48862 - llm.external-model-e2e-rt-sw-9dcd2a-914404c28491.2
2026-11-03T20:15:05.970304Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:15:26.710349Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T20:15:26.710381Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T20:15:26.710391Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
2026-11-03T20:15:26.710396Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T20:15:26.710554Z	error	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Task failed: Some("2")	thread=25
[2026-11-03T20:15:26.539Z] "POST /anything/e2e-rt-sw-9dcd2a-client HTTP/1.1" 200 - via_upstream - "-" 124 1577 171 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "369ea8ab-3e75-4ab7-9af6-8d2ff06fa236" "httpbingo.org" "66.241.125.232:443" outbound|443||httpbingo.org 10.134.0.37:33866 10.134.0.37:443 10.132.0.13:60310 - llm.external-model-e2e-rt-sw-9dcd2a-914404c28491.0
2026-11-03T20:15:26.936027Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T20:15:26.936055Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T20:15:26.936064Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
2026-11-03T20:15:26.936068Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T20:15:26.936208Z	error	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Task failed: Some("2")	thread=25
[2026-11-03T20:15:26.769Z] "POST /anything/e2e-rt-sw-9dcd2a-client HTTP/1.1" 200 - via_upstream - "-" 124 1577 166 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "b42f89bc-46d0-412d-ad6e-cf84079cc04c" "httpbingo.org" "66.241.125.232:443" outbound|443||httpbingo.org 10.134.0.37:33866 10.134.0.37:443 10.132.0.13:60320 - llm.external-model-e2e-rt-sw-9dcd2a-914404c28491.0
2026-11-03T20:15:36.782665Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:15:53.246388Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:16:22.403571Z	info	ads	ADS: "" 3 terminated with stream closed
2026-11-03T20:16:22.852950Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:16:26.185502Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:16:49.226457Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:17:10.786075Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:17:32.952284Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:18:35.952311Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Tracing span buffer full (100), dropping oldest spans to prevent memory growth	thread=25
2026-11-03T20:18:35.952496Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T20:18:35.952586Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T20:18:35.952630Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
2026-11-03T20:18:35.952638Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
[2026-11-03T20:18:35.814Z] "DELETE /v1/api-keys/6beff088-326f-4e0b-b4d1-6e58213dc126 HTTP/1.1" 200 - via_upstream - "-" 0 321 138 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "c7edb086-798f-4ada-a5f9-f46e67ac7b41" "maas.apps.96089cf8aa8b05683edc.prod.konflux-ocp-ci.dev" "10.134.0.29:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.134.0.37:57448 10.134.0.37:443 10.132.0.13:44692 - odh-ai-gateway-infra.maas-api-route.3
2026-11-03T20:18:39.311774Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T20:18:39.311807Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T20:18:39.311820Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T20:18:39.311825Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
[2026-11-03T20:18:39.205Z] "DELETE /v1/api-keys/4b50b42b-907a-44c4-a553-eaa668bdfe6c HTTP/1.1" 200 - via_upstream - "-" 0 321 106 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "20ad0527-33e6-4ed1-ba7c-3f11b332d08d" "maas.apps.96089cf8aa8b05683edc.prod.konflux-ocp-ci.dev" "10.134.0.29:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.134.0.37:57458 10.134.0.37:443 10.132.0.13:44712 - odh-ai-gateway-infra.maas-api-route.3
[2026-11-03T20:18:40.310Z] "POST /maas-api/v1/api-keys HTTP/1.1" 403 -  - "-" 77 0 40 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "05963fb5-2dce-49ad-9b48-1feccdee9ed1" "maas.apps.96089cf8aa8b05683edc.prod.konflux-ocp-ci.dev" "-" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local - 10.134.0.37:443 10.132.0.13:44724 - odh-ai-gateway-infra.maas-api-route.3
2026-11-03T20:18:51.037273Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T20:18:51.037364Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T20:18:51.037397Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T20:18:51.037418Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
[2026-11-03T20:18:50.397Z] "POST /v1/api-keys HTTP/1.1" 201 - via_upstream - "-" 93 317 639 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "7b2fe030-c5e1-4275-b9cb-18298117b0ff" "maas.apps.96089cf8aa8b05683edc.prod.konflux-ocp-ci.dev" "10.134.0.29:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.134.0.37:57458 10.134.0.37:443 10.132.0.13:43070 - odh-ai-gateway-infra.maas-api-route.3
2026-11-03T20:18:56.648126Z	warning	envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:372		thread=25
2026-11-03T20:18:56.648151Z	warning	envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:372		thread=25
2026-11-03T20:18:56.648155Z	warning	envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:372		thread=25
2026-11-03T20:18:56.648160Z	warning	envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:385	key 'model' missing from lines 1-1 key: model	thread=25
[2026-11-03T20:18:56.520Z] "POST /llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions HTTP/1.1" 403 -  - "-" 76 77 127 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "2bcd183d-242a-4220-a6bf-4b05b63b4e74" "maas.apps.96089cf8aa8b05683edc.prod.konflux-ocp-ci.dev" "-" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local - 10.134.0.37:443 10.132.0.13:53980 - v1-completions-path
2026-11-03T20:19:14.676077Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-03T20:24:07.152Z] "POST /v1/chat/completions HTTP/1.1" 200 - via_upstream - "-" 135 3079 77 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "d9b4806f-2136-4975-85cb-48ed46c4d28c" "maas.apps.96089cf8aa8b05683edc.prod.konflux-ocp-ci.dev" "10.134.0.35:8000" outbound|8000||facebook-opt-125m-simulated-kserve-workload-svc.llm.svc.cluster.local 10.134.0.37:45180 10.134.0.37:443 10.132.0.13:40574 - v1-chat-completions-path
2026-11-03T20:33:14.399241Z	info	ads	ADS: new connection for node:4
2026-11-03T20:33:14.418754Z	info	cache	read certificate from file	resource=file-root:system
2026-11-03T20:38:55.178679Z	info	xdsproxy	connected to delta upstream XDS server: istiod-openshift-gateway.openshift-ingress.svc:15012	id=2
2026-11-03T20:40:06.480126Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:40:43.479439Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:40:54.708245Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T20:40:54.708281Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T20:40:54.708292Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T20:40:54.708296Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
2026-11-03T20:40:54.708463Z	error	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Task failed: Some("2")	thread=25
[2026-11-03T20:40:54.570Z] "POST /anything/e2e-dft-b1808e-c4c89a HTTP/1.1" 200 - via_upstream - "-" 109 1594 137 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "ffb95fc3-6e94-47c3-b9e0-4581897efded" "httpbingo.org" "66.241.125.232:443" outbound|443||httpbingo.org 10.134.0.37:45592 10.134.0.37:443 10.132.0.13:36924 - llm.external-model-e2e-dft-b1808e-4c90157ebe32.0
2026-11-03T20:41:07.102041Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:41:09.741744Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:41:10.436883Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:41:16.897249Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:41:22.891861Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:41:54.884483Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:41:55.819189Z	info	ads	ADS: new connection for node:5
2026-11-03T20:41:55.819377Z	info	cache	adding watcher for file certificate /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
2026-11-03T20:41:55.819401Z	info	cache	read certificate from file	resource=file-root:/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
2026-11-03T20:42:00.101907Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:42:36.997116Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:43:13.888040Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:43:35.196252Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:43:47.598470Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:43:48.453663Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:43:55.451512Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:46:46.005416Z	info	ads	ADS: "" 5 terminated
2026-11-03T20:51:11.050170Z	info	ads	ADS: "" 4 terminated with stream closed
2026-11-03T20:52:08.664869Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:53:47.583776Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:54:58.662300Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:56:35.997911Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T20:57:09.611282Z	info	xdsproxy	connected to delta upstream XDS server: istiod-openshift-gateway.openshift-ingress.svc:15012	id=3
2026-11-03T20:59:59.118395Z	info	cache	event for file certificate /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt : REMOVE, pushing to proxy
2026-11-03T21:06:48.826208Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:06:48.860696Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T21:06:48.860742Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T21:06:48.860763Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T21:06:48.860771Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
[2026-11-03T21:06:48.788Z] "POST /v1/api-keys HTTP/1.1" 201 - via_upstream - "-" 70 313 72 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "ba2b4623-d3f5-4502-99ae-664607fe597d" "maas.apps.96089cf8aa8b05683edc.prod.konflux-ocp-ci.dev" "10.134.0.29:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.134.0.37:33906 10.134.0.37:443 10.132.0.13:46084 - odh-ai-gateway-infra.maas-api-route.3
2026-11-03T21:07:12.018915Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:07:46.289383Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:07:49.410562Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T21:07:49.410619Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T21:07:49.410640Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T21:07:49.410646Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
[2026-11-03T21:07:49.154Z] "POST /v1/api-keys HTTP/1.1" 201 - via_upstream - "-" 61 303 255 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "c11eaca8-94b3-4064-82d1-e37f8d629799" "maas.apps.96089cf8aa8b05683edc.prod.konflux-ocp-ci.dev" "10.134.0.29:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.134.0.37:51184 10.134.0.37:443 10.132.0.13:40198 - odh-ai-gateway-infra.maas-api-route.3
[2026-11-03T21:07:49.472Z] "POST /v1/chat/completions HTTP/1.1" 200 - via_upstream - "-" 147 412 87 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "5edafe6f-44e4-4619-8321-7a74c8b419bb" "maas.apps.96089cf8aa8b05683edc.prod.konflux-ocp-ci.dev" "10.133.0.37:8000" outbound|8000||echo-simulator-kserve-workload-svc.llm.svc.cluster.local 10.134.0.37:51070 10.134.0.37:443 10.132.0.13:40210 - v1-chat-completions-path
2026-11-03T21:08:02.248117Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:08:04.158431Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:08:10.438161Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:08:26.188576Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:08:27.956436Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:08:31.131245Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T21:08:31.131294Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T21:08:31.131317Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
2026-11-03T21:08:31.131324Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
[2026-11-03T21:08:30.892Z] "POST /v1/api-keys HTTP/1.1" 201 - via_upstream - "-" 73 315 239 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "32d619c6-6bb3-4ddc-adc6-184e34835af0" "maas.apps.96089cf8aa8b05683edc.prod.konflux-ocp-ci.dev" "10.134.0.29:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.134.0.37:51184 10.134.0.37:443 10.132.0.13:51566 - odh-ai-gateway-infra.maas-api-route.3
2026-11-03T21:08:42.911154Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:08:53.843906Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:08:56.302865Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:08:56.353260Z	info	ads	ADS: new connection for node:6
2026-11-03T21:08:56.353654Z	info	cache	adding watcher for file certificate /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
2026-11-03T21:08:56.353747Z	info	cache	read certificate from file	resource=file-root:/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
2026-11-03T21:08:56.879245Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:09:11.670689Z	info	ads	ADS: "" 6 terminated
2026-11-03T21:09:11.719565Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:09:12.151653Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:09:15.556002Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:09:15.980698Z	info	ads	ADS: new connection for node:7
2026-11-03T21:09:15.980901Z	info	cache	read certificate from file	resource=file-root:/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
2026-11-03T21:09:17.962215Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:09:40.616887Z	info	xdsproxy	connected to delta upstream XDS server: istiod-openshift-gateway.openshift-ingress.svc:15012	id=4
2026-11-03T21:09:56.623934Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-03T21:10:00.983Z] "POST /v1/chat/completions HTTP/1.1" 200 - via_upstream - "-" 131 357 93 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "2b7697b9-006c-46c3-a3de-c6b894082351" "maas.apps.96089cf8aa8b05683edc.prod.konflux-ocp-ci.dev" "10.133.0.188:8000" outbound|54321||e2e-prio-pool-0704dddb-inference-pool-ip-461f2a7f.llm.svc.cluster.local 10.134.0.37:49760 10.134.0.37:443 10.132.0.13:57746 - v1-model-routing
2026-11-03T21:10:16.046420Z	info	ads	ADS: "" 7 terminated
2026-11-03T21:10:16.170261Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:10:18.079956Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:10:20.740212Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:10:25.466714Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:12:39.207700Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:12:55.128436Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:13:10.755312Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:13:22.438719Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:13:26.525184Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:13:37.807608Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:13:40.451319Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:13:55.940882Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:14:40.531751Z	info	ads	ADS: new connection for node:8
2026-11-03T21:14:40.535691Z	info	cache	read certificate from file	resource=file-root:system
2026-11-03T21:14:41.066156Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:14:42.947315Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:14:48.481768Z	warning	envoy ext_proc external/envoy/source/extensions/filters/http/ext_proc/ext_proc.cc:1688	[Tags: "ConnectionId":"997","StreamId":"17277090484170467227"] Received gRPC error on stream: 14, message upstream connect error or disconnect/reset before headers. reset reason: remote connection failure, transport failure reason: delayed connect error: Connection refused{upstream_reset_before_response_started{remote_connection_failure|delayed_connect_error:_Connection_refused}}	thread=25
[2026-11-03T21:14:48.389Z] "POST /models-as-a-service/e2e-rotation-external-model/anything/v1/chat/completions HTTP/1.1" 500 - ext_proc_error_gRPC_error_14{upstream_connect_error_or_disconnect/reset_before_headers._reset_reason:_remote_connection_failure,_transport_failure_reason:_delayed_connect_error:_Connection_refused{upstream_reset_before_response_started{remote_connection_failure|delayed_connect_error:_Connection_refused}}} - "-" 92 0 92 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "257ed019-0be4-45c6-9bc1-265c6421d45d" "maas.apps.96089cf8aa8b05683edc.prod.konflux-ocp-ci.dev" "-" outbound|443||provider-selection-required-e2e-rotation-external-model.models-as-a-service.svc.cluster.local - 10.134.0.37:443 10.132.0.13:40478 - models-as-a-service.external-model-e2e-rotation-external-model.2
2026-11-03T21:14:50.605291Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Tracing span buffer full (100), dropping oldest spans to prevent memory growth	thread=24
2026-11-03T21:14:50.606315Z	warning	envoy ext_proc external/envoy/source/extensions/filters/http/ext_proc/ext_proc.cc:1688	[Tags: "ConnectionId":"1000","StreamId":"3434402845639948334"] Received gRPC error on stream: 14, message upstream connect error or disconnect/reset before headers. reset reason: remote connection failure, transport failure reason: delayed connect error: Connection refused{upstream_reset_before_response_started{remote_connection_failure|delayed_connect_error:_Connection_refused}}	thread=24
[2026-11-03T21:14:50.547Z] "POST /models-as-a-service/e2e-rotation-external-model/anything/v1/chat/completions HTTP/1.1" 500 - ext_proc_error_gRPC_error_14{upstream_connect_error_or_disconnect/reset_before_headers._reset_reason:_remote_connection_failure,_transport_failure_reason:_delayed_connect_error:_Connection_refused{upstream_reset_before_response_started{remote_connection_failure|delayed_connect_error:_Connection_refused}}} - "-" 92 0 59 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "6147fc72-8606-4268-884d-cc3946e0bade" "maas.apps.96089cf8aa8b05683edc.prod.konflux-ocp-ci.dev" "-" outbound|443||provider-selection-required-e2e-rotation-external-model.models-as-a-service.svc.cluster.local - 10.134.0.37:443 10.132.0.13:40488 - models-as-a-service.external-model-e2e-rotation-external-model.2
2026-11-03T21:15:38.771188Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T21:15:38.771222Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T21:15:38.771233Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T21:15:38.771238Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
2026-11-03T21:15:38.771401Z	error	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Task failed: Some("2")	thread=25
[2026-11-03T21:15:38.649Z] "POST /anything/v1/chat/completions HTTP/1.1" 200 - via_upstream - "-" 92 1535 121 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "84988d3d-24da-4dc0-850d-4231e67edf21" "httpbingo.org" "66.241.125.232:443" outbound|443||httpbingo.org 10.134.0.37:39264 10.134.0.37:443 10.132.0.13:60808 - models-as-a-service.external-model-e2e-rotation-external-model.0
2026-11-03T21:15:58.240830Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T21:15:58.240866Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T21:15:58.240877Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
2026-11-03T21:15:58.240882Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T21:15:58.241052Z	error	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Task failed: Some("2")	thread=25
[2026-11-03T21:15:58.145Z] "POST /anything/v1/chat/completions HTTP/1.1" 200 - via_upstream - "-" 92 1529 95 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "a2145815-53bb-4fd0-92be-2799150ca7cf" "httpbingo.org" "66.241.125.232:443" outbound|443||httpbingo.org 10.134.0.37:39264 10.134.0.37:443 10.132.0.13:60672 - models-as-a-service.external-model-e2e-rotation-external-model.0
2026-11-03T21:16:06.583975Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T21:16:06.584025Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T21:16:06.584043Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T21:16:06.584050Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
[2026-11-03T21:16:06.450Z] "DELETE /v1/api-keys/4e677a85-7012-4a6f-b693-df0a5ab0bf8a HTTP/1.1" 200 - via_upstream - "-" 0 359 133 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "fcb8e3e4-ce77-4a6f-a902-d19454090a78" "maas.apps.96089cf8aa8b05683edc.prod.konflux-ocp-ci.dev" "10.134.0.29:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.134.0.37:38174 10.134.0.37:443 10.132.0.13:40966 - odh-ai-gateway-infra.maas-api-route.3
2026-11-03T21:16:17.903177Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:16:18.054282Z	info	ads	ADS: "" 8 terminated with stream closed
2026-11-03T21:16:18.727704Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:16:22.005007Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:16:25.393837Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:16:36.748482Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:16:38.691684Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-03T21:16:42.184Z] "POST /llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions HTTP/1.1" 429 -  - "-" 76 18 57 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "874a0df1-604c-4758-8146-92717c842361" "maas.apps.96089cf8aa8b05683edc.prod.konflux-ocp-ci.dev" "-" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local - 10.134.0.37:443 10.132.0.13:40784 - v1-completions-path
2026-11-03T21:16:54.015840Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:16:55.965325Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:16:59.424985Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:17:06.861050Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-03T21:17:13.823Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 295 71 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "efe23870-8fad-433e-8152-37beba4c0500" "maas.apps.96089cf8aa8b05683edc.prod.konflux-ocp-ci.dev" "10.134.0.34:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.134.0.37:60994 10.134.0.37:443 10.132.0.13:54630 - v1-completions-path
2026-11-03T21:17:31.001584Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:17:41.282645Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:17:43.202479Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:18:02.120875Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:18:04.061836Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:18:10.110679Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:18:21.014045Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:18:34.719452Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:18:36.531993Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:18:42.498772Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:19:00.161243Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:19:03.438935Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:19:18.754873Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:19:21.208670Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:19:23.567524Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T21:19:23.567561Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T21:19:23.567576Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
2026-11-03T21:19:23.567580Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
[2026-11-03T21:19:23.510Z] "DELETE /v1/api-keys/606f1555-5e80-488a-88eb-37a5ced8a135 HTTP/1.1" 200 - via_upstream - "-" 0 375 57 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "d4db9167-7522-4c12-9a2d-c62af2c0ea0b" "maas.apps.96089cf8aa8b05683edc.prod.konflux-ocp-ci.dev" "10.134.0.29:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.134.0.37:59898 10.134.0.37:443 10.132.0.13:58072 - odh-ai-gateway-infra.maas-api-route.3
2026-11-03T21:19:28.548644Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:19:46.840568Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:19:57.103283Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:20:13.489995Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:20:16.640421Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:24:04.108845Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:24:19.211790Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:24:21.050031Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:24:28.307569Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:24:46.489550Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:24:50.277118Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:25:05.981018Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:25:16.234080Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:25:18.726092Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:25:25.882770Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T21:25:25.882804Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T21:25:25.882818Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T21:25:25.882823Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
2026-11-03T21:25:25.994351Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T21:25:25.994383Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T21:25:25.994396Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T21:25:25.994401Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
[2026-11-03T21:25:25.818Z] "POST /v1/api-keys HTTP/1.1" 201 - via_upstream - "-" 67 309 64 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "6a37bbd4-3769-4a84-9a1d-e02af37575f3" "maas.apps.96089cf8aa8b05683edc.prod.konflux-ocp-ci.dev" "10.134.1.30:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.134.0.37:52302 10.134.0.37:443 10.132.0.13:35270 - odh-ai-gateway-infra.maas-api-route.3
[2026-11-03T21:25:25.936Z] "POST /v1/api-keys HTTP/1.1" 201 - via_upstream - "-" 67 310 58 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "d800030e-d091-4521-b056-4c1a876eb83b" "maas.apps.96089cf8aa8b05683edc.prod.konflux-ocp-ci.dev" "10.134.1.30:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.134.0.37:52302 10.134.0.37:443 10.132.0.13:35278 - odh-ai-gateway-infra.maas-api-route.3
2026-11-03T21:25:27.450079Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T21:25:27.450114Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T21:25:27.450128Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
2026-11-03T21:25:27.450133Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
[2026-11-03T21:25:27.393Z] "DELETE /v1/api-keys/6fe9c982-0edc-4f83-9421-39c46ab0cab9 HTTP/1.1" 200 - via_upstream - "-" 0 365 56 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "400380e8-e33c-4d8d-bf57-7ce514bf8142" "maas.apps.96089cf8aa8b05683edc.prod.konflux-ocp-ci.dev" "10.134.1.30:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.134.0.37:52302 10.134.0.37:443 10.132.0.13:35296 - odh-ai-gateway-infra.maas-api-route.3
2026-11-03T21:25:39.222344Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:25:41.613189Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:25:45.606231Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-03T21:25:49.807Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 290 69 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "9f590490-270e-43e9-b76d-6f678566a294" "maas.apps.96089cf8aa8b05683edc.prod.konflux-ocp-ci.dev" "10.134.0.34:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.134.0.37:43518 10.134.0.37:443 10.132.0.13:39612 - v1-completions-path
[2026-11-03T21:25:52.088Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 292 61 - "44.212.242.249,10.132.0.13" "python-requests/2.34.2" "40852b4f-1ed5-44e4-9ff9-42cff9304097" "maas.apps.96089cf8aa8b05683edc.prod.konflux-ocp-ci.dev" "10.134.0.34:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.134.0.37:60994 10.134.0.37:443 10.132.0.13:39616 - v1-completions-path
2026-11-03T21:26:06.605127Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:26:16.823635Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:26:19.238509Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:26:23.195648Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:26:28.616930Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:26:47.182506Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:26:57.340475Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:26:59.269254Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:27:09.216518Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:27:13.102550Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:27:17.878813Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:27:35.341146Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:27:55.080516Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:28:05.390714Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:28:15.487903Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:28:25.746340Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:28:35.942952Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:28:46.281571Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:28:56.330799Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:29:06.535635Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:29:16.722121Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T21:29:26.958893Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8