2026-11-03T01:20:32.832924Z	info	FLAG: --concurrency="0"
2026-11-03T01:20:32.832945Z	info	FLAG: --domain="openshift-ingress.svc.cluster.local"
2026-11-03T01:20:32.832949Z	info	FLAG: --help="false"
2026-11-03T01:20:32.832952Z	info	FLAG: --log_as_json="false"
2026-11-03T01:20:32.832954Z	info	FLAG: --log_caller=""
2026-11-03T01:20:32.832956Z	info	FLAG: --log_output_level="default:info"
2026-11-03T01:20:32.832958Z	info	FLAG: --log_stacktrace_level="default:none"
2026-11-03T01:20:32.832963Z	info	FLAG: --log_target="[stdout]"
2026-11-03T01:20:32.832965Z	info	FLAG: --meshConfig="./etc/istio/config/mesh"
2026-11-03T01:20:32.832967Z	info	FLAG: --outlierLogPath=""
2026-11-03T01:20:32.832970Z	info	FLAG: --profiling="true"
2026-11-03T01:20:32.832978Z	info	FLAG: --proxyComponentLogLevel="misc:error"
2026-11-03T01:20:32.832981Z	info	FLAG: --proxyLogLevel="warning"
2026-11-03T01:20:32.832983Z	info	FLAG: --serviceCluster="istio-proxy"
2026-11-03T01:20:32.832985Z	info	FLAG: --stsPort="0"
2026-11-03T01:20:32.832987Z	info	FLAG: --templateFile=""
2026-11-03T01:20:32.832989Z	info	FLAG: --tokenManagerPlugin=""
2026-11-03T01:20:32.832995Z	info	FLAG: --vklog="0"
2026-11-03T01:20:32.832998Z	info	Version 1.28.10-cebc47d567fd1333707199224c18129ecfe07c05-Clean
2026-11-03T01:20:32.833150Z	info	Proxy role	ips=[10.134.1.41] type=router id=maas-default-gateway-openshift-default-596dc9545f-kr275.openshift-ingress domain=openshift-ingress.svc.cluster.local
2026-11-03T01:20:32.833198Z	info	Apply proxy config from env {"discoveryAddress":"istiod-openshift-gateway.openshift-ingress.svc:15012","proxyHeaders":{"server":{"disabled":true},"envoyDebugHeaders":{"disabled":true},"metadataExchangeHeaders":{"mode":"IN_MESH"}}}

2026-11-03T01:20:32.834690Z	info	cpu limit detected as 2, setting concurrency
2026-11-03T01:20:32.834998Z	info	Effective config: binaryPath: /usr/local/bin/envoy
concurrency: 2
configPath: ./etc/istio/proxy
controlPlaneAuthPolicy: MUTUAL_TLS
discoveryAddress: istiod-openshift-gateway.openshift-ingress.svc:15012
drainDuration: 45s
proxyAdminPort: 15000
proxyHeaders:
  envoyDebugHeaders:
    disabled: true
  metadataExchangeHeaders:
    mode: IN_MESH
  server:
    disabled: true
serviceCluster: istio-proxy
statNameLength: 189
statusPort: 15020
terminationDrainDuration: 5s

2026-11-03T01:20:32.835015Z	info	JWT policy is third-party-jwt
2026-11-03T01:20:32.835019Z	info	using credential fetcher of JWT type in cluster.local trust domain
2026-11-03T01:20:32.836864Z	info	Starting default Istio SDS Server
2026-11-03T01:20:32.836896Z	info	CA Endpoint istiod-openshift-gateway.openshift-ingress.svc:15012, provider Citadel
2026-11-03T01:20:32.836921Z	info	Using CA istiod-openshift-gateway.openshift-ingress.svc:15012 cert with certs: var/run/secrets/istio/root-cert.pem
2026-11-03T01:20:32.837386Z	info	Opening status port 15020
2026-11-03T01:20:32.838145Z	info	xdsproxy	Initializing with upstream address "istiod-openshift-gateway.openshift-ingress.svc:15012" and cluster "Kubernetes"
2026-11-03T01:20:32.839067Z	info	sds	Starting SDS grpc server
2026-11-03T01:20:32.839087Z	info	sds	Starting SDS server for workload certificates, will listen on "var/run/secrets/workload-spiffe-uds/socket"
2026-11-03T01:20:32.839837Z	info	Pilot SAN: [istiod-openshift-gateway.openshift-ingress.svc]
2026-11-03T01:20:32.840710Z	info	Starting proxy agent
2026-11-03T01:20:32.840747Z	info	Envoy command: [-c etc/istio/proxy/envoy-rev.json --drain-time-s 45 --drain-strategy immediate --local-address-ip-version v4 --file-flush-interval-msec 1000 --disable-hot-restart --allow-unknown-static-fields -l warning --component-log-level misc:error --skip-deprecated-logs --concurrency 2]
2026-11-03T01:20:32.870569Z	info	cache	generated new workload certificate	resourceName=default latency=31.326372ms ttl=23h59m59.129434867s
2026-11-03T01:20:32.870604Z	info	cache	Root cert has changed, start rotating root cert
2026-11-03T01:20:32.870637Z	info	cache	returned workload trust anchor from cache	ttl=23h59m59.129364262s
2026-11-03T01:20:32.909445Z	info	xdsproxy	connected to delta upstream XDS server: istiod-openshift-gateway.openshift-ingress.svc:15012	id=1
2026-11-03T01:20:33.138198Z	info	ads	ADS: new connection for node:1
2026-11-03T01:20:33.143697Z	info	cache	adding watcher for file certificate /etc/pki/tls/certs/ca-bundle.crt
2026-11-03T01:20:33.143742Z	info	cache	read certificate from file	resource=file-root:system
2026-11-03T01:20:33.175887Z	info	ads	ADS: new connection for node:2
2026-11-03T01:20:33.175971Z	info	cache	returned workload certificate from cache	ttl=23h59m58.824032023s
2026-11-03T01:20:33.177058Z	info	ads	ADS: new connection for node:3
2026-11-03T01:20:33.177127Z	info	cache	returned workload trust anchor from cache	ttl=23h59m58.822874247s
2026-11-03T01:20:33.283819Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:20:43.543919Z	info	Readiness succeeded in 10.713930849s
2026-11-03T01:20:43.544517Z	info	Envoy proxy is ready
2026-11-03T01:20:43.704528Z	info	Readiness succeeded in 10.874547091s
2026-11-03T01:20:56.925875Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:21:32.053381Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:21:52.837644Z	info	ads	ADS: new connection for node:4
2026-11-03T01:21:52.837936Z	info	cache	adding watcher for file certificate /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
2026-11-03T01:21:52.839073Z	info	cache	read certificate from file	resource=file-root:/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
2026-11-03T01:21:55.313346Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:22:00.778633Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T01:22:00.778684Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T01:22:00.778707Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
2026-11-03T01:22:00.778714Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
[2026-11-03T01:22:00.325Z] "POST /v1/api-keys HTTP/1.1" 201 - via_upstream - "-" 66 308 453 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "8cef0e12-0a91-4419-bae7-c956863350b6" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.133.0.48:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.134.1.41:54238 10.134.1.41:443 10.132.0.9:44608 - odh-ai-gateway-infra.maas-api-route.3
2026-11-03T01:22:06.165101Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T01:22:06.165219Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T01:22:06.165267Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
2026-11-03T01:22:06.165331Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T01:22:06.165666Z	error	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Task failed: Some("2")	thread=25
[2026-11-03T01:22:06.014Z] "POST /anything/e2e-dft-44821e-3c70c2 HTTP/1.1" 200 - via_upstream - "-" 109 1772 151 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "71e6fcbe-0878-41b9-b692-1e82daf7e22c" "httpbingo.org" "66.241.125.232:443" outbound|443||httpbingo.org 10.134.1.41:39882 10.134.1.41:443 10.132.0.9:44624 - llm.external-model-e2e-dft-44821e-613a0b3f174b.0
[2026-11-03T01:22:06.511Z] "POST /llm/e2e-dft-44821e/anything/e2e-dft-44821e-3c70c2 HTTP/1.1" 401 -  - "-" 109 88 45 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "143d2b35-f9f9-4704-8862-9838638342de" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "-" outbound|443||provider-selection-required-e2e-dft-44821e.llm.svc.cluster.local - 10.134.1.41:443 10.132.0.9:44636 - llm.external-model-e2e-dft-44821e-613a0b3f174b.2
2026-11-03T01:22:18.306974Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:22:36.071091Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:25:45.982357Z	info	ads	ADS: "" 4 terminated
2026-11-03T01:26:46.613237Z	info	ads	ADS: "" 1 terminated with stream closed
2026-11-03T01:29:04.839962Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:30:05.951750Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:34:49.601257Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:36:29.426760Z	info	xdsproxy	connected to delta upstream XDS server: istiod-openshift-gateway.openshift-ingress.svc:15012	id=2
2026-11-03T01:36:34.795324Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:44:16.721397Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:44:17.194088Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T01:44:17.194150Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T01:44:17.194176Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
2026-11-03T01:44:17.194185Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
[2026-11-03T01:44:17.083Z] "POST /v1/api-keys HTTP/1.1" 201 - via_upstream - "-" 70 313 111 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "96153a51-b6f6-4651-8a47-33762a5f7ac3" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.133.0.48:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.134.1.41:46728 10.134.1.41:443 10.132.0.9:45306 - odh-ai-gateway-infra.maas-api-route.3
2026-11-03T01:44:39.522771Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:45:14.312690Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:45:29.031159Z	warning	envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:372		thread=24
2026-11-03T01:45:29.031185Z	warning	envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:372		thread=24
2026-11-03T01:45:29.031189Z	warning	envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:372		thread=24
2026-11-03T01:45:29.031192Z	warning	envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:385	key 'model' missing from lines 1-1 key: model	thread=24
2026-11-03T01:45:29.406335Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-03T01:45:28.985Z] "POST /llm/e2e-embedding-simulated/v1/embeddings HTTP/1.1" 403 -  - "-" 61 103 45 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "b0403582-6c45-48e4-a901-3115dc91d020" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "-" outbound|8000||e2e-embedding-simulated-kserve-workload-svc.llm.svc.cluster.local - 10.134.1.41:443 10.132.0.9:34646 - v1-catch-all-path
2026-11-03T01:45:31.185461Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:45:33.742090Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T01:45:33.742126Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T01:45:33.742141Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T01:45:33.742147Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
[2026-11-03T01:45:33.660Z] "POST /v1/api-keys HTTP/1.1" 201 - via_upstream - "-" 81 323 82 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "a20a7383-bf99-407b-91a1-b1260542074c" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.133.0.48:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.134.1.41:56400 10.134.1.41:443 10.132.0.9:44362 - odh-ai-gateway-infra.maas-api-route.3
2026-11-03T01:45:35.121551Z	warning	envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:385	key 'completion_tokens' missing from lines 1-1 key: completion_tokens	thread=24
[2026-11-03T01:45:35.039Z] "POST /v1/embeddings HTTP/1.1" 200 - via_upstream - "-" 61 4185 81 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "66d76020-c242-4d26-ba54-c476e58606f4" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.132.0.38:8000" outbound|8000||e2e-embedding-simulated-kserve-workload-svc.llm.svc.cluster.local 10.134.1.41:52954 10.134.1.41:443 10.132.0.9:44372 - v1-catch-all-path
2026-11-03T01:45:37.027029Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-03T01:45:40.576Z] "POST /llm/e2e-embedding-simulated/v1/embeddings HTTP/1.1" 429 -  - "-" 61 18 58 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "275e9f4a-5d7d-485b-9fe0-991a37fb41b2" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "-" outbound|8000||e2e-embedding-simulated-kserve-workload-svc.llm.svc.cluster.local - 10.134.1.41:443 10.132.0.9:41364 - v1-catch-all-path
2026-11-03T01:45:52.058412Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:45:53.613688Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:45:58.899076Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:46:18.961964Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:46:20.626059Z	info	ads	ADS: new connection for node:5
2026-11-03T01:46:20.626951Z	info	cache	read certificate from file	resource=file-root:/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
2026-11-03T01:46:21.605666Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:47:02.814182Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-03T01:47:06.653Z] "POST /v1/chat/completions HTTP/1.1" 200 - via_upstream - "-" 131 348 89 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "ec4ce81a-edef-407a-8f27-20f14fe3f1f2" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.132.1.107:8000" outbound|54321||e2e-prio-pool-55b7e383-inference-pool-ip-b446f665.llm.svc.cluster.local 10.134.1.41:56708 10.134.1.41:443 10.132.0.9:37914 - v1-model-routing
2026-11-03T01:47:09.387104Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Tracing span buffer full (100), dropping oldest spans to prevent memory growth	thread=24
[2026-11-03T01:47:09.332Z] "POST /v1/chat/completions HTTP/1.1" 200 - via_upstream - "-" 131 329 63 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "1f61ad5c-dfc9-46e4-8556-ea823eec3fc4" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.132.1.107:8000" outbound|54321||e2e-prio-pool-55b7e383-inference-pool-ip-b446f665.llm.svc.cluster.local 10.134.1.41:56714 10.134.1.41:443 10.132.0.9:46018 - v1-model-routing
2026-11-03T01:47:21.323162Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:47:21.675429Z	info	ads	ADS: "" 5 terminated with stream closed
2026-11-03T01:47:23.063215Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:47:25.781180Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:47:29.535941Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:49:44.045504Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T01:49:44.045539Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T01:49:44.045553Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
2026-11-03T01:49:44.045558Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
[2026-11-03T01:49:43.973Z] "GET /v1/models HTTP/1.1" 200 - via_upstream - "-" 0 27 72 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "c45d61ea-ee9b-48fb-8195-fb2f4ca50a63" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.133.0.48:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.134.1.41:48084 10.134.1.41:443 10.132.0.9:50726 - odh-ai-gateway-infra.maas-api-route.3
2026-11-03T01:49:50.693773Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:50:03.907122Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:50:20.012911Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:50:31.785099Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:50:35.177172Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:50:46.162189Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:50:47.996228Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-03T01:50:51.076Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 292 83 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "50fb1348-add7-43ce-998c-09281ed5fa61" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.132.0.39:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.134.1.41:49488 10.134.1.41:443 10.132.0.9:34094 - v1-completions-path
2026-11-03T01:51:02.983980Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-03T01:51:46.890Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 66 282 83 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "933f4c8a-ae91-4ecd-8815-efc1c0fedca7" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.132.0.40:8000" outbound|8000||facebook-opt-125m-simulated-kserve-workload-svc.llm.svc.cluster.local 10.134.1.41:52992 10.134.1.41:443 10.132.0.9:56346 - v1-completions-path
2026-11-03T01:51:48.323900Z	info	ads	ADS: new connection for node:6
2026-11-03T01:51:48.328878Z	info	cache	read certificate from file	resource=file-root:system
2026-11-03T01:51:48.789695Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:51:50.095301Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:51:54.976599Z	warning	envoy ext_proc external/envoy/source/extensions/filters/http/ext_proc/ext_proc.cc:1688	[Tags: "ConnectionId":"513","StreamId":"15064357739206213025"] Received gRPC error on stream: 14, message upstream connect error or disconnect/reset before headers. reset reason: remote connection failure, transport failure reason: delayed connect error: Connection refused{upstream_reset_before_response_started{remote_connection_failure|delayed_connect_error:_Connection_refused}}	thread=25
[2026-11-03T01:51:54.916Z] "POST /models-as-a-service/e2e-rotation-external-model/anything/v1/chat/completions HTTP/1.1" 500 - ext_proc_error_gRPC_error_14{upstream_connect_error_or_disconnect/reset_before_headers._reset_reason:_remote_connection_failure,_transport_failure_reason:_delayed_connect_error:_Connection_refused{upstream_reset_before_response_started{remote_connection_failure|delayed_connect_error:_Connection_refused}}} - "-" 92 0 60 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "aac47179-888e-48fe-a521-d95db52b4ef5" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "-" outbound|443||provider-selection-required-e2e-rotation-external-model.models-as-a-service.svc.cluster.local - 10.134.1.41:443 10.132.0.9:57962 - models-as-a-service.external-model-e2e-rotation-external-model.2
2026-11-03T01:52:05.875101Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T01:52:05.875137Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T01:52:05.875148Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
2026-11-03T01:52:05.875152Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T01:52:05.875354Z	error	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Task failed: Some("2")	thread=24
[2026-11-03T01:52:05.745Z] "POST /anything/v1/chat/completions HTTP/1.1" 200 - via_upstream - "-" 92 1714 129 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "2dd0de04-7227-48f7-8e8a-d1590bfcaabc" "httpbingo.org" "66.241.125.232:443" outbound|443||httpbingo.org 10.134.1.41:60004 10.134.1.41:443 10.132.0.9:38766 - models-as-a-service.external-model-e2e-rotation-external-model.0
2026-11-03T01:52:36.295149Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Tracing span buffer full (100), dropping oldest spans to prevent memory growth	thread=25
2026-11-03T01:52:36.372071Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T01:52:36.372114Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T01:52:36.372132Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T01:52:36.372138Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
2026-11-03T01:52:36.372423Z	error	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Task failed: Some("2")	thread=25
[2026-11-03T01:52:36.234Z] "POST /anything/v1/chat/completions HTTP/1.1" 200 - via_upstream - "-" 92 1716 137 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "7f7b49c9-7a1f-4d39-9e7e-bf23ea015cc1" "httpbingo.org" "66.241.125.232:443" outbound|443||httpbingo.org 10.134.1.41:51526 10.134.1.41:443 10.132.0.9:42220 - models-as-a-service.external-model-e2e-rotation-external-model.0
2026-11-03T01:52:40.712114Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T01:52:40.712152Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T01:52:40.712164Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
2026-11-03T01:52:40.712168Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T01:52:40.712388Z	error	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Task failed: Some("2")	thread=25
[2026-11-03T01:52:40.615Z] "POST /anything/v1/chat/completions HTTP/1.1" 200 - via_upstream - "-" 92 1708 97 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "71d06d40-fa21-432c-bffe-2771669026fb" "httpbingo.org" "66.241.125.232:443" outbound|443||httpbingo.org 10.134.1.41:51526 10.134.1.41:443 10.132.0.9:45082 - models-as-a-service.external-model-e2e-rotation-external-model.0
2026-11-03T01:52:45.013520Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T01:52:45.013554Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T01:52:45.013566Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
2026-11-03T01:52:45.013571Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T01:52:45.013765Z	error	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Task failed: Some("2")	thread=24
[2026-11-03T01:52:44.915Z] "POST /anything/v1/chat/completions HTTP/1.1" 200 - via_upstream - "-" 92 1717 97 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "c8d7bd44-4327-4b47-a8b5-8dba7498aff5" "httpbingo.org" "66.241.125.232:443" outbound|443||httpbingo.org 10.134.1.41:60004 10.134.1.41:443 10.132.0.9:45096 - models-as-a-service.external-model-e2e-rotation-external-model.0
2026-11-03T01:53:06.688996Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T01:53:06.689037Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T01:53:06.689050Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
2026-11-03T01:53:06.689055Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T01:53:06.689260Z	error	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Task failed: Some("2")	thread=24
[2026-11-03T01:53:06.592Z] "POST /anything/v1/chat/completions HTTP/1.1" 200 - via_upstream - "-" 92 1717 96 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "10f1a242-0950-49dd-a2ee-b1b345c31928" "httpbingo.org" "66.241.125.232:443" outbound|443||httpbingo.org 10.134.1.41:60004 10.134.1.41:443 10.132.0.9:47304 - models-as-a-service.external-model-e2e-rotation-external-model.0
2026-11-03T01:53:28.258589Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:53:28.385847Z	info	ads	ADS: "" 6 terminated
2026-11-03T01:53:28.859057Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:53:29.207823Z	warning	envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:372		thread=25
2026-11-03T01:53:29.207856Z	warning	envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:372		thread=25
2026-11-03T01:53:29.207861Z	warning	envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:372		thread=25
2026-11-03T01:53:29.207867Z	warning	envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:385	key 'model' missing from lines 1-1 key: model	thread=25
[2026-11-03T01:53:29.153Z] "POST /llm/premium-simulated-simulated-premium/v1/completions HTTP/1.1" 403 -  - "-" 74 77 54 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "1fcef9ac-ad9d-4c85-9da2-af010a0bce49" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "-" outbound|8000||premium-simulated-simulated-premium-kserve-workload-svc.llm.svc.cluster.local - 10.134.1.41:443 10.132.0.9:56800 - v1-completions-path
2026-11-03T01:53:31.506102Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:53:34.571943Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:53:46.826411Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:53:48.436035Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:54:03.307791Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:54:04.864879Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:54:07.666756Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:54:15.965785Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-03T01:54:23.261Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 295 63 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "93e1d001-7042-4d35-97fc-37bb230bf19c" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.132.0.39:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.134.1.41:49488 10.134.1.41:443 10.132.0.9:41496 - v1-completions-path
[2026-11-03T01:54:25.565Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 297 63 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "7f64c7c5-43a6-4782-804e-651c1e42f29f" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.132.0.39:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.134.1.41:33482 10.134.1.41:443 10.132.0.9:41516 - v1-completions-path
2026-11-03T01:54:40.347646Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:54:50.563985Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:54:52.209825Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-03T01:54:58.358Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 290 64 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "0d17bb21-98d2-48ab-8050-a9a86b07d525" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.132.0.39:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.134.1.41:49488 10.134.1.41:443 10.132.0.9:32878 - v1-completions-path
[2026-11-03T01:54:58.967Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 294 57 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "e2c377ed-0a72-406a-b04d-0f88b4afaf61" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.132.0.39:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.134.1.41:33482 10.134.1.41:443 10.132.0.9:32888 - v1-completions-path
2026-11-03T01:55:11.261805Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:55:12.952496Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:55:27.440150Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:55:29.480910Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:55:35.036585Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-03T01:55:41.749Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 66 282 72 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "1eb0a433-f7ef-45c4-8dd9-6d2c3a68c036" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.132.0.40:8000" outbound|8000||facebook-opt-125m-simulated-kserve-workload-svc.llm.svc.cluster.local 10.134.1.41:46606 10.134.1.41:443 10.132.0.9:54500 - v1-completions-path
2026-11-03T01:55:43.729502Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:55:57.626254Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:56:08.229770Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:56:10.791198Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:56:11.487432Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T01:56:11.487486Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T01:56:11.487513Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T01:56:11.487521Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
[2026-11-03T01:56:11.395Z] "POST /v1/api-keys HTTP/1.1" 201 - via_upstream - "-" 81 324 91 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "42766cc5-53e1-42c6-8158-ef0711b3d71c" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.133.0.48:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.134.1.41:49334 10.134.1.41:443 10.132.0.9:60530 - odh-ai-gateway-infra.maas-api-route.3
2026-11-03T01:56:25.377989Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:56:27.451636Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-03T01:56:29.587Z] "POST /llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions HTTP/1.1" 403 -  - "-" 76 0 36 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "1981521d-cacd-473a-a3d1-6d5edfeefca5" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "-" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local - 10.134.1.41:443 10.132.0.9:41790 - v1-completions-path
2026-11-03T01:56:31.730286Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:56:38.303012Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T01:56:38.303050Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T01:56:38.303066Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
2026-11-03T01:56:38.303071Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
[2026-11-03T01:56:38.237Z] "POST /v1/api-keys HTTP/1.1" 201 - via_upstream - "-" 67 310 65 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "bf582fb4-d282-4139-8330-7c7fc2b7168f" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.133.0.48:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.134.1.41:49334 10.134.1.41:443 10.132.0.9:41814 - odh-ai-gateway-infra.maas-api-route.3
2026-11-03T01:56:50.219140Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:57:00.506044Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:57:16.908973Z	error	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: gRPC status code is not OK	thread=24
[2026-11-03T01:57:14.906Z] "POST /maas-api/v1/api-keys HTTP/1.1" 500 -  - "-" 71 23 2003 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "472e3763-269f-482f-96be-cff7851f7778" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "-" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local - 10.134.1.41:443 10.132.0.9:50536 - odh-ai-gateway-infra.maas-api-route.3
2026-11-03T01:59:36.948476Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T01:59:39.176064Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:01:02.990094Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:01:17.751926Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:01:19.339623Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:01:34.138621Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:01:43.582166Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:01:47.397058Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-03T02:01:47.778Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 292 91 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "5e7eb5de-613a-4f89-a66c-3290578f39ad" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.132.0.39:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.134.1.41:49488 10.134.1.41:443 10.132.0.9:54282 - v1-completions-path
[2026-11-03T02:01:48.187Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 290 1743 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "0fa4ca5e-383a-4e21-a3dd-f646815c83a5" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.132.0.39:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.134.1.41:49488 10.134.1.41:443 10.132.0.9:54290 - v1-completions-path
[2026-11-03T02:01:50.336Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 295 61 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "53940651-3fea-4c26-b2f3-d6ee068fa9b4" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.132.0.39:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.134.1.41:49488 10.134.1.41:443 10.132.0.9:58558 - v1-completions-path
2026-11-03T02:02:02.572345Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:02:13.229864Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:02:16.223830Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-03T02:02:20.968Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 290 64 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "1461e412-4907-4841-8e64-fcde1076a8c1" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.132.0.39:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.134.1.41:33482 10.134.1.41:443 10.132.0.9:37534 - v1-completions-path
2026-11-03T02:02:22.299494Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T02:02:22.299531Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T02:02:22.299548Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
2026-11-03T02:02:22.299553Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
[2026-11-03T02:02:22.236Z] "DELETE /v1/api-keys/42569afe-2067-4af9-bd54-07c7e2f0c619 HTTP/1.1" 200 - via_upstream - "-" 0 365 62 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "6b02f5ac-a08d-4b7f-8e2c-5d63c23a4b93" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.134.1.171:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.134.1.41:50900 10.134.1.41:443 10.132.0.9:37542 - odh-ai-gateway-infra.maas-api-route.3
2026-11-03T02:02:33.795639Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:02:35.857808Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:02:38.891383Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-03T02:02:43.626Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 293 63 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "a578d88e-abb0-434a-9b2d-4f0ddbd383c3" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.132.0.39:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.134.1.41:33482 10.134.1.41:443 10.132.0.9:56542 - v1-completions-path
[2026-11-03T02:02:43.969Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 290 62 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "afe8b061-8585-43ce-b43b-ca5ab47a8073" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.132.0.39:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.134.1.41:49488 10.134.1.41:443 10.132.0.9:56556 - v1-completions-path
[2026-11-03T02:02:45.530Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 290 61 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "62df29e4-6558-4810-aeea-d22f7d59eaf9" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.132.0.39:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.134.1.41:49488 10.134.1.41:443 10.132.0.9:56570 - v1-completions-path
[2026-11-03T02:02:47.042Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 290 62 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "b187ab90-c33e-41fd-80b6-efc6d394db69" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.132.0.39:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.134.1.41:49488 10.134.1.41:443 10.132.0.9:56576 - v1-completions-path
[2026-11-03T02:02:48.200Z] "POST /llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions HTTP/1.1" 429 -  - "-" 76 18 54 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "8fc9e129-4223-45df-a9c9-b4a4832faf0d" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "-" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local - 10.134.1.41:443 10.132.0.9:56578 - v1-completions-path
2026-11-03T02:02:48.497848Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T02:02:48.497888Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T02:02:48.497906Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-03T02:02:48.497911Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
[2026-11-03T02:02:48.437Z] "DELETE /v1/api-keys/e0500b2e-e165-48f8-ba09-e792c418bfd6 HTTP/1.1" 200 - via_upstream - "-" 0 365 60 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "c36a5584-548f-48e7-b0ce-8c646ebe8c85" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.134.1.171:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.134.1.41:44626 10.134.1.41:443 10.132.0.9:56580 - odh-ai-gateway-infra.maas-api-route.3
2026-11-03T02:03:00.104459Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:03:10.445827Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:03:13.388120Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:03:15.855117Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-03T02:03:18.696Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 294 104 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "e53d2919-c4fd-4476-9bf0-ed3ca39921e4" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.132.0.39:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.134.1.41:49488 10.134.1.41:443 10.132.0.9:34280 - v1-completions-path
2026-11-03T02:03:20.345239Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-03T02:03:24.238Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 292 66 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "8b9cee7a-67c2-4542-a65b-41aa571d7f67" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.132.0.39:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.134.1.41:33482 10.134.1.41:443 10.132.0.9:44132 - v1-completions-path
2026-11-03T02:03:25.361111Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T02:03:25.361149Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-03T02:03:25.361166Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
2026-11-03T02:03:25.361171Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
[2026-11-03T02:03:25.300Z] "DELETE /v1/api-keys/0e66eeda-3fde-46cd-a933-f71ce98d0dce HTTP/1.1" 200 - via_upstream - "-" 0 367 60 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "da8de74a-9e8d-4c8c-9ede-f5ffc0cbf335" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.134.1.171:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.134.1.41:59892 10.134.1.41:443 10.132.0.9:44134 - odh-ai-gateway-infra.maas-api-route.3
2026-11-03T02:03:36.896929Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:03:47.047531Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:03:48.659441Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:03:55.851765Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:03:59.155391Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:04:14.807841Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:04:34.689666Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:04:44.886463Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:04:55.155627Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:05:05.308110Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:05:15.543104Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:05:25.792578Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:05:35.962412Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:05:46.197105Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:05:56.424621Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-03T02:06:06.592884Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-03T02:06:13.476Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 66 285 78 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "2df10043-45dc-4082-bf5f-353197aba5e2" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.132.0.40:8000" outbound|8000||facebook-opt-125m-simulated-kserve-workload-svc.llm.svc.cluster.local 10.134.1.41:46606 10.134.1.41:443 10.132.0.9:38180 - v1-completions-path
[2026-11-03T02:06:14.000Z] "POST /llm/facebook-opt-125m-simulated/v1/completions HTTP/1.1" 403 -  - "-" 66 0 45 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "590bcd4c-3d45-418f-8420-57e42803b8d0" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "-" outbound|8000||facebook-opt-125m-simulated-kserve-workload-svc.llm.svc.cluster.local - 10.134.1.41:443 10.132.0.9:38186 - v1-completions-path
[2026-11-03T02:06:14.342Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 66 281 72 - "44.212.242.249,10.132.0.9" "python-requests/2.34.2" "7cfd8146-788c-420b-9b18-9c60298616a5" "maas.apps.8703189bab9016eab030.prod.konflux-ocp-ci.dev" "10.132.0.40:8000" outbound|8000||facebook-opt-125m-simulated-kserve-workload-svc.llm.svc.cluster.local 10.134.1.41:46606 10.134.1.41:443 10.132.0.9:38202 - v1-completions-path