2026-11-02T21:16:14.295106Z	info	FLAG: --concurrency="0"
2026-11-02T21:16:14.295124Z	info	FLAG: --domain="openshift-ingress.svc.cluster.local"
2026-11-02T21:16:14.295157Z	info	FLAG: --help="false"
2026-11-02T21:16:14.295160Z	info	FLAG: --log_as_json="false"
2026-11-02T21:16:14.295162Z	info	FLAG: --log_caller=""
2026-11-02T21:16:14.295164Z	info	FLAG: --log_output_level="default:info"
2026-11-02T21:16:14.295166Z	info	FLAG: --log_stacktrace_level="default:none"
2026-11-02T21:16:14.295170Z	info	FLAG: --log_target="[stdout]"
2026-11-02T21:16:14.295173Z	info	FLAG: --meshConfig="./etc/istio/config/mesh"
2026-11-02T21:16:14.295175Z	info	FLAG: --outlierLogPath=""
2026-11-02T21:16:14.295177Z	info	FLAG: --profiling="true"
2026-11-02T21:16:14.295179Z	info	FLAG: --proxyComponentLogLevel="misc:error"
2026-11-02T21:16:14.295180Z	info	FLAG: --proxyLogLevel="warning"
2026-11-02T21:16:14.295183Z	info	FLAG: --serviceCluster="istio-proxy"
2026-11-02T21:16:14.295185Z	info	FLAG: --stsPort="0"
2026-11-02T21:16:14.295187Z	info	FLAG: --templateFile=""
2026-11-02T21:16:14.295189Z	info	FLAG: --tokenManagerPlugin=""
2026-11-02T21:16:14.295191Z	info	FLAG: --vklog="0"
2026-11-02T21:16:14.295194Z	info	Version 1.28.10-cebc47d567fd1333707199224c18129ecfe07c05-Clean
2026-11-02T21:16:14.295345Z	info	Proxy role	ips=[10.132.0.33] type=router id=maas-default-gateway-openshift-default-fcb75c45f-4r6s4.openshift-ingress domain=openshift-ingress.svc.cluster.local
2026-11-02T21:16:14.295396Z	info	Apply proxy config from env {"discoveryAddress":"istiod-openshift-gateway.openshift-ingress.svc:15012","proxyHeaders":{"server":{"disabled":true},"envoyDebugHeaders":{"disabled":true},"metadataExchangeHeaders":{"mode":"IN_MESH"}}}

2026-11-02T21:16:14.296501Z	info	cpu limit detected as 2, setting concurrency
2026-11-02T21:16:14.296695Z	info	Effective config: binaryPath: /usr/local/bin/envoy
concurrency: 2
configPath: ./etc/istio/proxy
controlPlaneAuthPolicy: MUTUAL_TLS
discoveryAddress: istiod-openshift-gateway.openshift-ingress.svc:15012
drainDuration: 45s
proxyAdminPort: 15000
proxyHeaders:
  envoyDebugHeaders:
    disabled: true
  metadataExchangeHeaders:
    mode: IN_MESH
  server:
    disabled: true
serviceCluster: istio-proxy
statNameLength: 189
statusPort: 15020
terminationDrainDuration: 5s

2026-11-02T21:16:14.296707Z	info	JWT policy is third-party-jwt
2026-11-02T21:16:14.296711Z	info	using credential fetcher of JWT type in cluster.local trust domain
2026-11-02T21:16:14.298557Z	info	Starting default Istio SDS Server
2026-11-02T21:16:14.298581Z	info	CA Endpoint istiod-openshift-gateway.openshift-ingress.svc:15012, provider Citadel
2026-11-02T21:16:14.298613Z	info	Using CA istiod-openshift-gateway.openshift-ingress.svc:15012 cert with certs: var/run/secrets/istio/root-cert.pem
2026-11-02T21:16:14.298615Z	info	Opening status port 15020
2026-11-02T21:16:14.299851Z	info	xdsproxy	Initializing with upstream address "istiod-openshift-gateway.openshift-ingress.svc:15012" and cluster "Kubernetes"
2026-11-02T21:16:14.300472Z	info	sds	Starting SDS grpc server
2026-11-02T21:16:14.300491Z	info	sds	Starting SDS server for workload certificates, will listen on "var/run/secrets/workload-spiffe-uds/socket"
2026-11-02T21:16:14.301547Z	info	Pilot SAN: [istiod-openshift-gateway.openshift-ingress.svc]
2026-11-02T21:16:14.302904Z	info	Starting proxy agent
2026-11-02T21:16:14.302946Z	info	Envoy command: [-c etc/istio/proxy/envoy-rev.json --drain-time-s 45 --drain-strategy immediate --local-address-ip-version v4 --file-flush-interval-msec 1000 --disable-hot-restart --allow-unknown-static-fields -l warning --component-log-level misc:error --skip-deprecated-logs --concurrency 2]
2026-11-02T21:16:14.378298Z	info	xdsproxy	connected to delta upstream XDS server: istiod-openshift-gateway.openshift-ingress.svc:15012	id=1
2026-11-02T21:16:14.390495Z	info	cache	generated new workload certificate	resourceName=default latency=89.925144ms ttl=23h59m59.609509066s
2026-11-02T21:16:14.390544Z	info	cache	Root cert has changed, start rotating root cert
2026-11-02T21:16:14.390586Z	info	cache	returned workload trust anchor from cache	ttl=23h59m59.609415104s
2026-11-02T21:16:14.517848Z	info	ads	ADS: new connection for node:1
2026-11-02T21:16:14.517931Z	info	cache	returned workload certificate from cache	ttl=23h59m59.482070934s
2026-11-02T21:16:14.518276Z	info	ads	ADS: new connection for node:2
2026-11-02T21:16:14.518356Z	info	cache	returned workload trust anchor from cache	ttl=23h59m59.481644986s
2026-11-02T21:16:14.560593Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T21:16:20.585581Z	info	Readiness succeeded in 6.293269145s
2026-11-02T21:16:20.585908Z	info	Envoy proxy is ready
2026-11-02T21:17:40.010860Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T21:17:43.126317Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T21:18:16.776157Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T21:18:21.582673Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T21:18:27.089789Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T21:18:31.648744Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T21:18:39.098056Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T21:18:48.915631Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-02T21:19:22.588Z] "POST /v1/chat/completions HTTP/1.1" 200 - via_upstream - "-" 122 448 79 - "34.228.250.194,10.132.0.20" "curl/7.76.1" "8e4b9121-09a9-4d08-aa6f-d3c5eb43fa9c" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.133.0.37:8000" outbound|8000||facebook-opt-125m-simulated-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:55858 10.132.0.33:443 10.132.0.20:55560 - v1-model-routing
2026-11-02T21:25:06.756608Z	info	ads	ADS: new connection for node:3
2026-11-02T21:25:06.762590Z	info	cache	adding watcher for file certificate /etc/pki/tls/certs/ca-bundle.crt
2026-11-02T21:25:06.762705Z	info	cache	read certificate from file	resource=file-root:system
2026-11-02T21:25:11.092930Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T21:25:24.924704Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T21:25:29.710929Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T21:25:29.710971Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T21:25:29.710982Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
2026-11-02T21:25:29.710987Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T21:25:29.711155Z	error	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Task failed: Some("2")	thread=24
[2026-11-02T21:25:29.640Z] "POST /llm/e2e-rt-sw-847782/anything/e2e-rt-sw-847782-client HTTP/1.1" 503 UH no_healthy_upstream - "-" 124 19 70 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "0f56cace-3add-447d-b2a1-214b3f5524da" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "-" outbound|443||provider-selection-required-e2e-rt-sw-847782.llm.svc.cluster.local - 10.132.0.33:443 10.132.0.20:44738 - llm.external-model-e2e-rt-sw-847782-931a1687e4da.2
2026-11-02T21:25:39.315851Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T21:25:45.925748Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T21:25:45.925790Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T21:25:45.925805Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T21:25:45.925812Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
2026-11-02T21:25:45.925994Z	error	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Task failed: Some("2")	thread=25
[2026-11-02T21:25:45.856Z] "POST /llm/e2e-rt-sw-847782/anything/e2e-rt-sw-847782-client HTTP/1.1" 503 UH no_healthy_upstream - "-" 124 19 69 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "22ce8868-d0d9-4d51-88d5-049a25313bc5" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "-" outbound|443||provider-selection-required-e2e-rt-sw-847782.llm.svc.cluster.local - 10.132.0.33:443 10.132.0.20:52368 - llm.external-model-e2e-rt-sw-847782-931a1687e4da.2
2026-11-02T21:26:11.484489Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T21:26:25.763192Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T21:26:45.273787Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-02T21:26:54.651Z] "POST /llm/e2e-rt-eq-d2da5a/anything/e2e-rt-eq-d2da5a-client HTTP/1.1" 503 -  - "-" 124 0 98 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "1fe07eeb-a697-4df3-b9a9-179869da564a" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "-" outbound|443||provider-selection-required-e2e-rt-eq-d2da5a.llm.svc.cluster.local - 10.132.0.33:443 10.132.0.20:42580 - llm.external-model-e2e-rt-eq-d2da5a-04e45017708d.4
2026-11-02T21:27:31.812664Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Tracing span buffer full (100), dropping oldest spans to prevent memory growth	thread=24
[2026-11-02T21:27:31.741Z] "POST /llm/e2e-rt-eq-d2da5a/anything/e2e-rt-eq-d2da5a-client HTTP/1.1" 503 -  - "-" 124 0 71 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "6de0034f-f7d4-47b3-90f0-1bc446556635" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "-" outbound|443||provider-selection-required-e2e-rt-eq-d2da5a.llm.svc.cluster.local - 10.132.0.33:443 10.132.0.20:42502 - llm.external-model-e2e-rt-eq-d2da5a-04e45017708d.4
[2026-11-02T21:27:52.873Z] "POST /llm/e2e-rt-eq-d2da5a/anything/e2e-rt-eq-d2da5a-client HTTP/1.1" 503 -  - "-" 124 0 68 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "9a339564-eee2-47d9-b411-e4cc56eaf874" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "-" outbound|443||provider-selection-required-e2e-rt-eq-d2da5a.llm.svc.cluster.local - 10.132.0.33:443 10.132.0.20:49362 - llm.external-model-e2e-rt-eq-d2da5a-04e45017708d.4
[2026-11-02T21:29:07.995Z] "POST /llm/e2e-rt-eq-d2da5a/anything/e2e-rt-eq-d2da5a-client HTTP/1.1" 503 -  - "-" 124 0 65 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "69d89cf5-21b1-492a-8579-62855dd8e12f" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "-" outbound|443||provider-selection-required-e2e-rt-eq-d2da5a.llm.svc.cluster.local - 10.132.0.33:443 10.132.0.20:57032 - llm.external-model-e2e-rt-eq-d2da5a-04e45017708d.4
[2026-11-02T21:29:18.595Z] "POST /llm/e2e-rt-eq-d2da5a/anything/e2e-rt-eq-d2da5a-client HTTP/1.1" 503 -  - "-" 124 0 79 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "43c5141c-7107-4493-8c35-a49eb4f321e6" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "-" outbound|443||provider-selection-required-e2e-rt-eq-d2da5a.llm.svc.cluster.local - 10.132.0.33:443 10.132.0.20:57308 - llm.external-model-e2e-rt-eq-d2da5a-04e45017708d.4
2026-11-02T21:30:19.418228Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T21:30:24.031627Z	info	ads	ADS: "" 3 terminated with stream closed
2026-11-02T21:30:27.195169Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T21:30:45.147690Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T21:32:25.015820Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T21:35:41.567780Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T21:35:41.567828Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T21:35:41.567851Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T21:35:41.567859Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
[2026-11-02T21:35:41.393Z] "DELETE /v1/api-keys/ebca1765-50d7-4077-aefa-47641c9232e4 HTTP/1.1" 200 - via_upstream - "-" 0 321 174 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "234f5975-faaf-4963-a0af-f48b1e6177f9" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.132.0.49:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.132.0.33:47608 10.132.0.33:443 10.132.0.20:45692 - odh-ai-gateway-infra.maas-api-route.3
2026-11-02T21:35:43.944679Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T21:35:43.944714Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T21:35:43.944731Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
2026-11-02T21:35:43.944735Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
[2026-11-02T21:35:43.837Z] "POST /v1/api-keys HTTP/1.1" 201 - via_upstream - "-" 71 313 107 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "c20f4a4d-e913-4621-bd1d-0d2d7cb5ea0e" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.132.0.49:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.132.0.33:42490 10.132.0.33:443 10.132.0.20:45708 - odh-ai-gateway-infra.maas-api-route.3
2026-11-02T21:35:44.845200Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T21:35:44.845244Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T21:35:44.845266Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
2026-11-02T21:35:44.845273Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
[2026-11-02T21:35:44.738Z] "POST /v1/api-keys HTTP/1.1" 201 - via_upstream - "-" 71 313 107 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "a384240c-5c9d-4346-9854-2d431e53067b" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.132.0.49:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.132.0.33:42490 10.132.0.33:443 10.132.0.20:50976 - odh-ai-gateway-infra.maas-api-route.3
[2026-11-02T21:36:13.816Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 66 285 65 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "119fc4d5-0142-4ceb-9961-dc468d4f0e3c" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.133.0.37:8000" outbound|8000||facebook-opt-125m-simulated-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:55858 10.132.0.33:443 10.132.0.20:53650 - v1-completions-path
[2026-11-02T21:36:15.132Z] "POST /maas-api/internal/v1/api-keys/cleanup HTTP/1.1" 404 NR route_not_found - "-" 2 0 7 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "418b571d-30a5-4f4f-b68f-3aadcfc72db2" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "-" - - 10.132.0.33:443 10.132.0.20:53940 - -
2026-11-02T21:40:20.032357Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T21:40:49.233241Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-02T21:43:32.069Z] "POST /v1/chat/completions HTTP/1.1" 200 - via_upstream - "-" 110 326 82 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "761907fb-1414-4e2c-a21c-1340473ea164" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.133.0.37:8000" outbound|8000||facebook-opt-125m-simulated-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:49746 10.132.0.33:443 10.132.0.20:38080 - v1-chat-completions-path
2026-11-02T21:44:27.101066Z	info	xdsproxy	connected to delta upstream XDS server: istiod-openshift-gateway.openshift-ingress.svc:15012	id=2
[2026-11-02T21:49:21.631Z] "POST /v1/chat/completions HTTP/1.1" 200 - via_upstream - "-" 136 327 78 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "050ba3ae-7060-4d2c-81fd-7e4a343dbc84" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.133.0.37:8000" outbound|8000||facebook-opt-125m-simulated-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:55858 10.132.0.33:443 10.132.0.20:59946 - v1-chat-completions-path
2026-11-02T21:50:01.918912Z	info	xdsproxy	connected to delta upstream XDS server: istiod-openshift-gateway.openshift-ingress.svc:15012	id=3
[2026-11-02T22:02:36.919Z] "GET /v1/tenants HTTP/1.1" 404 NR route_not_found - "-" 0 0 9 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "8eeed7ab-13c0-4b05-ad03-3b2d08dd418b" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "-" - - 10.132.0.33:443 10.132.0.20:56510 - -
2026-11-02T22:09:12.183926Z	info	ads	ADS: new connection for node:4
2026-11-02T22:09:12.203349Z	info	cache	read certificate from file	resource=file-root:system
2026-11-02T22:09:16.347644Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:09:27.645175Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:09:46.584148Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:09:56.230747Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Tracing span buffer full (100), dropping oldest spans to prevent memory growth	thread=25
2026-11-02T22:09:56.252850Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T22:09:56.252887Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T22:09:56.252902Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T22:09:56.252907Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
[2026-11-02T22:09:56.004Z] "POST /v1/api-keys HTTP/1.1" 201 - via_upstream - "-" 66 308 248 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "447f2b62-d1f4-4ee0-89f3-46b6c377feb7" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.132.0.49:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.132.0.33:40316 10.132.0.33:443 10.132.0.20:52894 - odh-ai-gateway-infra.maas-api-route.3
2026-11-02T22:09:56.695957Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T22:09:56.695988Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T22:09:56.696002Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
2026-11-02T22:09:56.696007Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T22:09:56.696187Z	error	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Task failed: Some("2")	thread=25
2026-11-02T22:09:57.508868Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T22:09:57.508904Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T22:09:57.508915Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
2026-11-02T22:09:57.508920Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T22:09:57.509090Z	error	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Task failed: Some("2")	thread=24
[2026-11-02T22:09:56.463Z] "POST /anything/e2e-dft-55455f-ce6bf7 HTTP/1.1" 200 - via_upstream - "-" 109 1767 232 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "52cabd57-2a40-47bc-9579-7d86c6e66aec" "httpbingo.org" "66.241.125.232:443" outbound|443||httpbingo.org 10.132.0.33:48436 10.132.0.33:443 10.132.0.20:52898 - llm.external-model-e2e-dft-55455f-08db79027314.0
[2026-11-02T22:09:57.288Z] "POST /anything/e2e-dft-55455f-ce6bf7 HTTP/1.1" 200 - via_upstream - "-" 109 1768 220 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "13270889-229e-4859-916e-ceff8f032f13" "httpbingo.org" "66.241.125.232:443" outbound|443||httpbingo.org 10.132.0.33:48452 10.132.0.33:443 10.132.0.20:52912 - llm.external-model-e2e-dft-55455f-08db79027314.0
2026-11-02T22:09:58.190759Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T22:09:58.190792Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T22:09:58.190802Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T22:09:58.190806Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
2026-11-02T22:09:58.190960Z	error	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Task failed: Some("2")	thread=24
[2026-11-02T22:09:58.056Z] "POST /anything/e2e-dft-55455f-ce6bf7 HTTP/1.1" 200 - via_upstream - "-" 109 1763 134 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "0b34ed4c-2518-4881-852b-a5dfc9a59bb4" "httpbingo.org" "66.241.125.232:443" outbound|443||httpbingo.org 10.132.0.33:48452 10.132.0.33:443 10.132.0.20:52916 - llm.external-model-e2e-dft-55455f-08db79027314.0
2026-11-02T22:10:10.526646Z	info	ads	ADS: "" 4 terminated with stream closed
2026-11-02T22:10:11.467197Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:10:13.356535Z	info	ads	ADS: new connection for node:5
2026-11-02T22:10:13.365301Z	info	cache	read certificate from file	resource=file-root:system
2026-11-02T22:10:17.320607Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:10:19.132795Z	info	ads	ADS: "" 5 terminated with stream closed
2026-11-02T22:10:20.812211Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:10:25.645932Z	info	ads	ADS: new connection for node:6
2026-11-02T22:10:25.653523Z	info	cache	read certificate from file	resource=file-root:system
2026-11-02T22:10:27.993294Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:10:31.809543Z	info	ads	ADS: "" 6 terminated
2026-11-02T22:10:32.746272Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:10:34.476074Z	info	ads	ADS: new connection for node:7
2026-11-02T22:10:34.482320Z	info	cache	read certificate from file	resource=file-root:system
2026-11-02T22:10:35.701491Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:10:39.442843Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:10:40.347852Z	info	ads	ADS: "" 7 terminated
2026-11-02T22:10:42.894172Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:15:44.487988Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:16:59.684819Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:18:17.306062Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:19:46.467923Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:22:32.317847Z	info	xdsproxy	connected to delta upstream XDS server: istiod-openshift-gateway.openshift-ingress.svc:15012	id=4
2026-11-02T22:36:05.241693Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T22:36:05.241730Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T22:36:05.241746Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T22:36:05.241751Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
[2026-11-02T22:36:05.060Z] "POST /v1/api-keys HTTP/1.1" 201 - via_upstream - "-" 30 314 180 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "bc3c8f52-ceb6-4480-91a0-90947953b30d" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.132.0.49:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.132.0.33:51464 10.132.0.33:443 10.132.0.20:59626 - odh-ai-gateway-infra.maas-api-route.3
2026-11-02T22:36:10.434165Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T22:36:10.434201Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T22:36:10.434216Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T22:36:10.434222Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
2026-11-02T22:36:10.671508Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-02T22:36:10.325Z] "POST /v1/api-keys HTTP/1.1" 201 - via_upstream - "-" 70 312 109 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "d41d2a74-141e-4fb5-a6e2-ac496d2674ef" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.132.0.49:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.132.0.33:51478 10.132.0.33:443 10.132.0.20:59642 - odh-ai-gateway-infra.maas-api-route.3
2026-11-02T22:36:42.189472Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:36:51.602474Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T22:36:51.602517Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T22:36:51.602529Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T22:36:51.602534Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
[2026-11-02T22:36:51.495Z] "POST /v1/api-keys HTTP/1.1" 400 - via_upstream - "-" 80 94 106 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "6df6613d-d57a-4d4e-8b88-2f2cc1989bb0" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.132.0.49:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.132.0.33:51478 10.132.0.33:443 10.132.0.20:60734 - odh-ai-gateway-infra.maas-api-route.3
2026-11-02T22:37:17.245537Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-02T22:37:20.013Z] "POST /v1/chat/completions HTTP/1.1" 200 - via_upstream - "-" 2817 3089 76 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "bc45c914-b971-95ee-af89-f9d0204e9c42" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.133.0.36:8000" outbound|8000||echo-simulator-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:42170 10.132.0.33:443 10.132.0.20:35256 - v1-chat-completions-path
[2026-11-02T22:37:20.286Z] "POST /v1/chat/completions HTTP/1.1" 200 - via_upstream - "-" 108118 108394 321 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "f26bcf9b-28be-4933-a21e-26792e222960" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.133.0.36:8000" outbound|8000||echo-simulator-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:42174 10.132.0.33:443 10.132.0.20:35262 - v1-chat-completions-path
[2026-11-02T22:37:21.258Z] "POST /v1/chat/completions HTTP/1.1" 200 - via_upstream - "-" 108118 108394 278 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "28e792f8-c19a-4660-a683-17e434fca27a" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.133.0.36:8000" outbound|8000||echo-simulator-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:42170 10.132.0.33:443 10.132.0.20:35268 - v1-chat-completions-path
2026-11-02T22:37:35.926377Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:37:40.848857Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:37:43.762955Z	warning	envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:385	key 'completion_tokens' missing from lines 1-1 key: completion_tokens	thread=25
[2026-11-02T22:37:43.274Z] "POST /v1/embeddings HTTP/1.1" 200 - via_upstream - "-" 28720 3466266 515 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "c0397902-5841-4419-9b19-b536a807caa1" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.133.0.34:8000" outbound|8000||e2e-embedding-simulated-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:39042 10.132.0.33:443 10.132.0.20:55990 - v1-catch-all-path
2026-11-02T22:37:45.092780Z	warning	envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:385	key 'completion_tokens' missing from lines 1-1 key: completion_tokens	thread=25
[2026-11-02T22:37:44.968Z] "POST /v1/embeddings HTTP/1.1" 200 - via_upstream - "-" 91 3616 124 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "0055c1e9-fb59-4739-bf63-598febc66587" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.133.0.34:8000" outbound|8000||e2e-embedding-simulated-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:39042 10.132.0.33:443 10.132.0.20:40794 - v1-catch-all-model-routing
2026-11-02T22:37:51.271710Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:37:53.907603Z	warning	envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:385	key 'completion_tokens' missing from lines 1-1 key: completion_tokens	thread=24
[2026-11-02T22:37:53.828Z] "POST /v1/embeddings HTTP/1.1" 200 - via_upstream - "-" 61 4185 78 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "1e5099bb-c463-452f-b7a2-2656149e4df5" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.133.0.34:8000" outbound|8000||e2e-embedding-simulated-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:46304 10.132.0.33:443 10.132.0.20:40820 - v1-catch-all-path
2026-11-02T22:37:54.313579Z	warning	envoy filter external/envoy/source/extensions/filters/http/json_to_metadata/filter.cc:385	key 'completion_tokens' missing from lines 1-1 key: completion_tokens	thread=24
[2026-11-02T22:37:54.218Z] "POST /v1/embeddings HTTP/1.1" 200 - via_upstream - "-" 61 4185 94 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "c1a21a41-806c-447f-99e9-b7e54ea9ec9c" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.133.0.34:8000" outbound|8000||e2e-embedding-simulated-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:46304 10.132.0.33:443 10.132.0.20:35888 - v1-catch-all-path
2026-11-02T22:38:04.897620Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:38:14.252964Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:38:24.197156Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:38:42.857154Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:38:44.866324Z	info	ads	ADS: new connection for node:8
2026-11-02T22:38:44.866508Z	info	cache	adding watcher for file certificate /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
2026-11-02T22:38:44.866544Z	info	cache	read certificate from file	resource=file-root:/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
2026-11-02T22:38:48.038748Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:39:28.631626Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-02T22:39:31.615Z] "POST /v1/chat/completions HTTP/1.1" 200 - via_upstream - "-" 131 331 93 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "8ae5d293-38bf-49dd-9de3-b341a89f2657" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.132.0.154:8000" outbound|54321||e2e-prio-pool-58e3ef14-inference-pool-ip-a6af34d1.llm.svc.cluster.local 10.132.0.33:44092 10.132.0.33:443 10.132.0.20:34404 - v1-model-routing
2026-11-02T22:39:51.900370Z	info	ads	ADS: "" 8 terminated
2026-11-02T22:39:52.383832Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:39:57.358976Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:40:10.713543Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:42:44.293282Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:43:08.696794Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:43:28.321032Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:43:42.289988Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:44:00.890524Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:44:13.219083Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:44:18.347278Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-02T22:44:21.759Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 293 73 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "c593b50a-5088-4e34-88ac-a6415078356b" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.134.0.38:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:55044 10.132.0.33:443 10.132.0.20:35780 - v1-completions-path
[2026-11-02T22:44:23.608Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 295 59 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "54b1f038-e020-470e-ae2f-d82a6a987408" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.134.0.38:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:55044 10.132.0.33:443 10.132.0.20:35788 - v1-completions-path
[2026-11-02T22:44:24.682Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 294 59 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "14c977da-c25d-49bf-9ec0-15b349ab55d1" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.134.0.38:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:49808 10.132.0.33:443 10.132.0.20:46398 - v1-completions-path
[2026-11-02T22:44:24.957Z] "POST /llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions HTTP/1.1" 429 -  - "-" 76 18 53 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "49c055bd-a36b-44b1-99cb-3981d7c1da6b" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "-" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local - 10.132.0.33:443 10.132.0.20:46410 - v1-completions-path
2026-11-02T22:44:39.217811Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-02T22:44:41.858Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 66 284 66 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "fa904b38-d382-4d9f-9c69-cbc5f0a9feec" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.133.0.37:8000" outbound|8000||facebook-opt-125m-simulated-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:52134 10.132.0.33:443 10.132.0.20:47332 - v1-completions-path
2026-11-02T22:45:26.670876Z	info	ads	ADS: new connection for node:9
2026-11-02T22:45:26.675896Z	info	cache	read certificate from file	resource=file-root:system
2026-11-02T22:45:30.185476Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:45:31.701716Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:45:34.550045Z	warning	envoy ext_proc external/envoy/source/extensions/filters/http/ext_proc/ext_proc.cc:1688	[Tags: "ConnectionId":"1359","StreamId":"5769418052637728033"] Received gRPC error on stream: 14, message upstream connect error or disconnect/reset before headers. reset reason: remote connection failure, transport failure reason: delayed connect error: Connection refused{upstream_reset_before_response_started{remote_connection_failure|delayed_connect_error:_Connection_refused}}	thread=25
[2026-11-02T22:45:34.490Z] "POST /models-as-a-service/e2e-rotation-external-model/anything/v1/chat/completions HTTP/1.1" 500 - ext_proc_error_gRPC_error_14{upstream_connect_error_or_disconnect/reset_before_headers._reset_reason:_remote_connection_failure,_transport_failure_reason:_delayed_connect_error:_Connection_refused{upstream_reset_before_response_started{remote_connection_failure|delayed_connect_error:_Connection_refused}}} - "-" 92 0 59 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "bf50e473-b269-41d5-a37d-98b9a4f5a421" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "-" outbound|443||provider-selection-required-e2e-rotation-external-model.models-as-a-service.svc.cluster.local - 10.132.0.33:443 10.132.0.20:47650 - models-as-a-service.external-model-e2e-rotation-external-model.2
2026-11-02T22:45:41.394844Z	warning	envoy ext_proc external/envoy/source/extensions/filters/http/ext_proc/ext_proc.cc:1688	[Tags: "ConnectionId":"1364","StreamId":"13965708468205420074"] Received gRPC error on stream: 14, message upstream connect error or disconnect/reset before headers. reset reason: remote connection failure, transport failure reason: delayed connect error: Connection refused{upstream_reset_before_response_started{remote_connection_failure|delayed_connect_error:_Connection_refused}}	thread=24
[2026-11-02T22:45:41.339Z] "POST /models-as-a-service/e2e-rotation-external-model/anything/v1/chat/completions HTTP/1.1" 500 - ext_proc_error_gRPC_error_14{upstream_connect_error_or_disconnect/reset_before_headers._reset_reason:_remote_connection_failure,_transport_failure_reason:_delayed_connect_error:_Connection_refused{upstream_reset_before_response_started{remote_connection_failure|delayed_connect_error:_Connection_refused}}} - "-" 92 0 56 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "454b1534-f72c-45a2-bdcc-9049e7361f6b" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "-" outbound|443||provider-selection-required-e2e-rotation-external-model.models-as-a-service.svc.cluster.local - 10.132.0.33:443 10.132.0.20:47674 - models-as-a-service.external-model-e2e-rotation-external-model.2
2026-11-02T22:46:00.478234Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T22:46:00.478268Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T22:46:00.478280Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T22:46:00.478284Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
2026-11-02T22:46:00.478441Z	error	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Task failed: Some("2")	thread=24
[2026-11-02T22:46:00.281Z] "POST /anything/v1/chat/completions HTTP/1.1" 200 - via_upstream - "-" 92 1711 196 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "425a425f-6801-489d-a5c4-2bd988672d6f" "httpbingo.org" "66.241.125.232:443" outbound|443||httpbingo.org 10.132.0.33:44306 10.132.0.33:443 10.132.0.20:53866 - models-as-a-service.external-model-e2e-rotation-external-model.0
2026-11-02T22:46:48.539019Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T22:46:48.539052Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T22:46:48.539062Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T22:46:48.539066Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
2026-11-02T22:46:48.539253Z	error	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Task failed: Some("2")	thread=25
[2026-11-02T22:46:48.339Z] "POST /anything/v1/chat/completions HTTP/1.1" 200 - via_upstream - "-" 92 1709 199 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "20448076-7849-4f4e-ad7c-fb18664633bf" "httpbingo.org" "66.241.125.232:443" outbound|443||httpbingo.org 10.132.0.33:33634 10.132.0.33:443 10.132.0.20:49932 - models-as-a-service.external-model-e2e-rotation-external-model.0
2026-11-02T22:46:55.621597Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T22:46:55.621628Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T22:46:55.621638Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
2026-11-02T22:46:55.621643Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T22:46:55.621812Z	error	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1155	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Task failed: Some("2")	thread=25
[2026-11-02T22:46:55.495Z] "POST /anything/v1/chat/completions HTTP/1.1" 200 - via_upstream - "-" 92 1710 126 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "94a7cf69-67df-4eb2-9a8c-6337c397572d" "httpbingo.org" "66.241.125.232:443" outbound|443||httpbingo.org 10.132.0.33:33634 10.132.0.33:443 10.132.0.20:40588 - models-as-a-service.external-model-e2e-rotation-external-model.0
2026-11-02T22:47:17.711631Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:47:17.736527Z	info	ads	ADS: "" 9 terminated with stream closed
2026-11-02T22:47:18.006244Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T22:47:18.006279Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T22:47:18.006295Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
2026-11-02T22:47:18.006300Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
[2026-11-02T22:47:17.890Z] "POST /v1/api-keys HTTP/1.1" 201 - via_upstream - "-" 74 316 115 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "607b51d4-155d-4a06-80c6-767f27eb5b20" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.132.0.49:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.132.0.33:53938 10.132.0.33:443 10.132.0.20:40014 - odh-ai-gateway-infra.maas-api-route.3
2026-11-02T22:47:18.671029Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:47:23.739042Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:47:29.376775Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-02T22:47:33.784Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 290 64 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "0c3992c6-99a0-42de-9140-d54b39a1ff29" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.134.0.38:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:55044 10.132.0.33:443 10.132.0.20:39952 - v1-completions-path
[2026-11-02T22:47:35.234Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 292 58 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "a5c6f3f2-2380-4f3e-af13-74fce9d68c65" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.134.0.38:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:55044 10.132.0.33:443 10.132.0.20:54256 - v1-completions-path
[2026-11-02T22:47:37.743Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 293 59 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "44536456-a36e-4b39-b8fd-54fdb9e698b9" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.134.0.38:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:49808 10.132.0.33:443 10.132.0.20:54272 - v1-completions-path
2026-11-02T22:47:52.170329Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:47:57.037233Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-02T22:48:00.366Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 295 60 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "0af11a3d-f12d-4586-889c-090dd3a6275f" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.134.0.38:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:55044 10.132.0.33:443 10.132.0.20:34132 - v1-completions-path
[2026-11-02T22:48:00.881Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 297 54 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "49dbd143-039e-449a-8f40-0c65a81574ea" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.134.0.38:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:55044 10.132.0.33:443 10.132.0.20:34148 - v1-completions-path
[2026-11-02T22:48:01.129Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 289 54 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "47ccafc7-8d17-4150-8635-e767e29f27ad" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.134.0.38:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:55044 10.132.0.33:443 10.132.0.20:34150 - v1-completions-path
[2026-11-02T22:48:02.018Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 295 56 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "d8d0eb5e-94d7-484b-9217-e64c7ca23acd" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.134.0.38:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:49808 10.132.0.33:443 10.132.0.20:34162 - v1-completions-path
[2026-11-02T22:48:03.784Z] "POST /llm/e2e-unconfigured-facebook-opt-125m-simulated/v1/completions HTTP/1.1" 429 -  - "-" 76 18 56 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "95cd0a95-214a-499f-a2f3-72f355289f20" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "-" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local - 10.132.0.33:443 10.132.0.20:34176 - v1-completions-path
2026-11-02T22:48:21.396145Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:48:34.474453Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:48:41.041322Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T22:48:41.041359Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T22:48:41.041376Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
2026-11-02T22:48:41.041381Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
[2026-11-02T22:48:40.924Z] "POST /v1/api-keys HTTP/1.1" 201 - via_upstream - "-" 77 319 116 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "fa2b6dc7-344a-47e4-acde-a0c31d931be3" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.132.0.49:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.132.0.33:46258 10.132.0.33:443 10.132.0.20:55038 - odh-ai-gateway-infra.maas-api-route.3
[2026-11-02T22:48:41.368Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 293 95 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "1799b906-7cd5-44c7-b6ae-2e421f39fcfa" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.134.0.38:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:49808 10.132.0.33:443 10.132.0.20:55052 - v1-completions-path
[2026-11-02T22:48:42.346Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 293 59 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "10196a1b-30e0-4695-99de-aaa2717df713" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.134.0.38:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:49808 10.132.0.33:443 10.132.0.20:55060 - v1-completions-path
[2026-11-02T22:48:45.111Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 294 58 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "926c24ef-aeb3-4875-b8fb-8e16c42fd90d" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.134.0.38:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:49808 10.132.0.33:443 10.132.0.20:50568 - v1-completions-path
[2026-11-02T22:48:46.159Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 292 63 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "f9fbf378-16d2-46c7-ba02-122796061e41" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.134.0.38:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:49808 10.132.0.33:443 10.132.0.20:50574 - v1-completions-path
2026-11-02T22:49:05.009042Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:49:15.727488Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:49:20.552461Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-02T22:49:33.098Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 290 58 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "6cdff45a-0253-4525-b6b8-92eb999e1c5e" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.134.0.38:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:55044 10.132.0.33:443 10.132.0.20:38806 - v1-completions-path
2026-11-02T22:49:50.287030Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:49:55.070149Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:50:08.819906Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:50:18.316681Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:50:25.914485Z	info	cache	event for file certificate /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt : REMOVE, pushing to proxy
2026-11-02T22:50:29.124865Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:50:35.790613Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:50:50.646895Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:51:03.492301Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-02T22:51:08.408Z] "POST /llm/facebook-opt-125m-simulated/v1/completions HTTP/1.1" 429 -  - "-" 66 18 62 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "0ba899b4-af3f-4bc9-8486-bed874e03184" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "-" outbound|8000||facebook-opt-125m-simulated-kserve-workload-svc.llm.svc.cluster.local - 10.132.0.33:443 10.132.0.20:45614 - v1-completions-path
2026-11-02T22:51:09.492476Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:51:29.755179Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:51:35.338962Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:51:46.846585Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:51:58.693903Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T22:51:58.693939Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T22:51:58.693954Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T22:51:58.693960Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
[2026-11-02T22:51:58.580Z] "POST /v1/api-keys HTTP/1.1" 201 - via_upstream - "-" 65 308 113 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "0b2fd488-6208-4d0e-950d-22206524e6a1" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.132.0.49:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.132.0.33:42274 10.132.0.33:443 10.132.0.20:51928 - odh-ai-gateway-infra.maas-api-route.3
2026-11-02T22:52:12.933887Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:52:23.602485Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-02T22:52:39.947Z] "POST /llm/facebook-opt-125m-simulated/v1/completions HTTP/1.1" 403 -  - "-" 66 0 43 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "6b17f780-ae4e-4abe-acf8-9976e032a6cc" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "-" outbound|8000||facebook-opt-125m-simulated-kserve-workload-svc.llm.svc.cluster.local - 10.132.0.33:443 10.132.0.20:42744 - v1-completions-path
2026-11-02T22:52:42.433291Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:52:47.366185Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:53:46.440348Z	info	xdsproxy	connected to delta upstream XDS server: istiod-openshift-gateway.openshift-ingress.svc:15012	id=5
2026-11-02T22:55:21.314311Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:56:50.662953Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:57:07.620615Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:57:12.171950Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:57:23.459295Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:57:41.317582Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:57:47.669318Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:58:07.322901Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:58:17.886625Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:58:23.997189Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:58:51.017988Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:58:57.110703Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:59:02.817829Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-02T22:59:10.250Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 290 61 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "2c0f34ac-d1f8-4418-ae77-6edfa58f5b33" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.134.0.38:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:55044 10.132.0.33:443 10.132.0.20:57562 - v1-completions-path
[2026-11-02T22:59:10.507Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 292 60 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "9591ae6f-aec5-4854-bd32-10562b86f98a" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.134.0.38:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:49808 10.132.0.33:443 10.132.0.20:57578 - v1-completions-path
[2026-11-02T22:59:11.128Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 294 58 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "c8e49fef-1b88-4302-bb5e-4588b547ee32" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.134.0.38:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:49808 10.132.0.33:443 10.132.0.20:57588 - v1-completions-path
[2026-11-02T22:59:11.491Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 295 60 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "760dded9-a0b3-412c-9528-49f4fc8a8c1a" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.134.0.38:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:55044 10.132.0.33:443 10.132.0.20:57610 - v1-completions-path
[2026-11-02T22:59:15.688Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 290 64 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "f2d25931-26a0-4749-8418-f02d95cc59e4" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.134.0.38:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:49808 10.132.0.33:443 10.132.0.20:58656 - v1-completions-path
2026-11-02T22:59:19.652198Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T22:59:19.652235Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T22:59:19.652253Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=24
2026-11-02T22:59:19.652258Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=24
2026-11-02T22:59:19.977676Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T22:59:19.977712Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
2026-11-02T22:59:19.977728Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /usage/total_tokens	thread=25
2026-11-02T22:59:19.977733Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/context.cc:1152	wasm log kuadrant-wasm-shim kuadrant_wasm_shim: Missing json property: /model	thread=25
[2026-11-02T22:59:19.542Z] "DELETE /v1/api-keys/54d9b2f7-77bd-485b-9d28-5dd948a45881 HTTP/1.1" 200 - via_upstream - "-" 0 363 110 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "bce50cac-fd80-479b-b412-c23b89929272" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.132.0.159:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.132.0.33:48174 10.132.0.33:443 10.132.0.20:58672 - odh-ai-gateway-infra.maas-api-route.3
[2026-11-02T22:59:19.853Z] "DELETE /v1/api-keys/2918b75b-9d09-4945-a7f6-10b36e312cc6 HTTP/1.1" 200 - via_upstream - "-" 0 365 124 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "dbf230f2-66d4-4c26-ae9d-7c64d6c540fd" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.134.0.242:8443" outbound|8443||maas-api.odh-ai-gateway-infra.svc.cluster.local 10.132.0.33:51216 10.132.0.33:443 10.132.0.20:58684 - odh-ai-gateway-infra.maas-api-route.3
2026-11-02T22:59:33.396893Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:59:44.256423Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:59:50.558554Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T22:59:57.719495Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T23:00:08.546434Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
[2026-11-02T23:00:16.304Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 290 75 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "3f45818f-0206-4bee-9cb7-eb069baa7c3e" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.134.0.38:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:49808 10.132.0.33:443 10.132.0.20:48836 - v1-completions-path
[2026-11-02T23:00:17.759Z] "POST /v1/completions HTTP/1.1" 200 - via_upstream - "-" 76 291 59 - "34.228.250.194,10.132.0.20" "python-requests/2.34.2" "360c736a-3263-405c-8c6c-a0b54c4ffa2a" "maas.apps.1f35823228f5b77bf43a.prod.konflux-ocp-ci.dev" "10.134.0.38:8000" outbound|8000||e2e-unconfiab60ef4d3a239b5143b412cab04acac3-kserve-workload-svc.llm.svc.cluster.local 10.132.0.33:55044 10.132.0.33:443 10.132.0.20:48838 - v1-completions-path
2026-11-02T23:00:31.999580Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T23:00:42.604809Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T23:00:47.167872Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T23:01:05.200266Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T23:01:11.767306Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T23:01:31.789252Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T23:01:53.333152Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T23:02:15.394577Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T23:02:35.474197Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T23:02:46.372336Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T23:02:57.117119Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T23:03:07.818369Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T23:03:18.688672Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8
2026-11-02T23:03:29.253924Z	warning	envoy wasm external/envoy/source/extensions/common/wasm/wasm.cc:317	Wasm remote code fetch is unstable and may cause a crash	thread=8