tests/test_tenant_external_model_shared.py::TestTenantAuthIsolation::test_tenant_keys_authorized_only_on_their_own_gateway@tenant_ipp {"namespace": "ai-tenant-e2e-extmod-a-6ac03f", "pod": "payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz", "status": {"conditions": [{"lastProbeTime": null, "lastTransitionTime": "2026-11-02T13:03:51Z", "observedGeneration": 1, "status": "True", "type": "PodReadyToStartContainers"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T13:03:49Z", "observedGeneration": 1, "status": "True", "type": "Initialized"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T13:04:02Z", "observedGeneration": 1, "status": "True", "type": "Ready"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T13:04:02Z", "observedGeneration": 1, "status": "True", "type": "ContainersReady"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T13:03:49Z", "observedGeneration": 1, "status": "True", "type": "PodScheduled"}], "containerStatuses": [{"allocatedResources": {"cpu": "50m", "memory": "64Mi"}, "containerID": "cri-o://c7e02363915d6321f0510ce08deb32896aca9553f01575cf6fa8c55d8bd7cdee", "image": "quay.io/opendatahub/odh-praxis-extproc:odh-stable", "imageID": "quay.io/opendatahub/odh-praxis-extproc@sha256:8b7084478cc266b2373bbfb04190ed3ca6fe759428663bddbab63b9c3ad8f130", "lastState": {}, "name": "payload-processing", "ready": true, "resources": {"limits": {"cpu": "500m", "memory": "256Mi"}, "requests": {"cpu": "50m", "memory": "64Mi"}}, "restartCount": 0, "started": true, "state": {"running": {"startedAt": "2026-11-02T13:03:50Z"}}, "user": {"linux": {"gid": 0, "supplementalGroups": [0, 1000840000], "uid": 1000840000}}, "volumeMounts": [{"mountPath": "/etc/praxis", "name": "plugins-config-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/routing", "name": "routing-overlay-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/credentials", "name": "provider-credentials", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", "name": "kube-api-access-s62fp", "readOnly": true, "recursiveReadOnly": "Disabled"}]}], "hostIP": "10.0.133.239", "hostIPs": [{"ip": "10.0.133.239"}], "observedGeneration": 1, "phase": "Running", "podIP": "10.133.0.248", "podIPs": [{"ip": "10.133.0.248"}], "qosClass": "Burstable", "startTime": "2026-11-02T13:03:49Z"}} [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:03:50.652967Z  INFO praxis_extproc::fips: installed rustls crypto provider provider="openssl" provider_fips=false kernel_fips=Some(false) fips_required=false [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:03:50.660358Z  INFO praxis_ai_filters::routing::intelligent_route: intelligent_route: overlay snapshot initialized path=/etc/praxis/routing/routing-overlay.json contract_format=Envelope schema_version="1.0.0" accepted_revision="2815215b131963638d932f709a6c9f00cd90812adc9380c2d9aaece01bebb1a6" serving_revision="2815215b131963638d932f709a6c9f00cd90812adc9380c2d9aaece01bebb1a6" candidate_count=2 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:03:50.660673Z  INFO praxis_ai_filters::routing::overlay: intelligent_route: overlay file watcher started path=/etc/praxis/routing/routing-overlay.json debounce_ms=500 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:03:50.661369Z  INFO praxis_extproc: starting ExtProc server grpc=0.0.0.0:9004 health=0.0.0.0:50052 metrics=0.0.0.0:9090 filters=3 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:03:50.661396Z  INFO praxis_extproc: gRPC max decoding message size configured bytes=33554432 env="GRPC_MAX_RECV_MSG_SIZE" [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:03:50.661401Z  INFO praxis_extproc::tls: generating self-signed TLS certificate [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:03:50.661420Z  INFO praxis_extproc::health: health server listening address=0.0.0.0:50052 serving=true fips=false [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:03:50.662154Z  INFO praxis_extproc::metrics: metrics server listening address=0.0.0.0:9090 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:04:22.830599Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:04:28.228462Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:04:33.482657Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:04:38.741558Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:04:44.000734Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:04:49.255435Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:04:54.507244Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:05:00.210490Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:05:05.465448Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:05:10.710941Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:05:15.968842Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:05:21.226810Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:05:26.482689Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:05:31.889115Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:05:37.139104Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:05:42.392290Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:05:47.642806Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:05:52.896654Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:05:58.148765Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:06:03.556525Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:06:08.815872Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:06:14.093083Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:06:19.342036Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:06:24.589883Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:06:29.856556Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:06:35.301488Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:06:40.554423Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:06:45.803810Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:06:51.069156Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:06:56.322705Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:07:01.582334Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:07:06.985870Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:07:12.235695Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:07:17.485520Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:07:22.737168Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-7c78lqxmz/payload-processing] 2026-11-02T13:07:27.986165Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 {"namespace": "ai-tenant-e2e-extmod-b-3a78b4", "pod": "payload-processing-external-model-e2e-extmod-b-3a78b4-589cd7l2n", "status": {"conditions": [{"lastProbeTime": null, "lastTransitionTime": "2026-11-02T13:03:48Z", "observedGeneration": 1, "status": "True", "type": "PodReadyToStartContainers"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T13:03:47Z", "observedGeneration": 1, "status": "True", "type": "Initialized"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T13:03:59Z", "observedGeneration": 1, "status": "True", "type": "Ready"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T13:03:59Z", "observedGeneration": 1, "status": "True", "type": "ContainersReady"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T13:03:47Z", "observedGeneration": 1, "status": "True", "type": "PodScheduled"}], "containerStatuses": [{"allocatedResources": {"cpu": "50m", "memory": "64Mi"}, "containerID": "cri-o://fa8436d5e1f22aedb6ab38e1b09198cba75251677e6dd3e69338c343b0892129", "image": "quay.io/opendatahub/odh-praxis-extproc:odh-stable", "imageID": "quay.io/opendatahub/odh-praxis-extproc@sha256:8b7084478cc266b2373bbfb04190ed3ca6fe759428663bddbab63b9c3ad8f130", "lastState": {}, "name": "payload-processing", "ready": true, "resources": {"limits": {"cpu": "500m", "memory": "256Mi"}, "requests": {"cpu": "50m", "memory": "64Mi"}}, "restartCount": 0, "started": true, "state": {"running": {"startedAt": "2026-11-02T13:03:48Z"}}, "user": {"linux": {"gid": 0, "supplementalGroups": [0, 1000850000], "uid": 1000850000}}, "volumeMounts": [{"mountPath": "/etc/praxis", "name": "plugins-config-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/routing", "name": "routing-overlay-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/credentials", "name": "provider-credentials", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", "name": "kube-api-access-7fzwm", "readOnly": true, "recursiveReadOnly": "Disabled"}]}], "hostIP": "10.0.133.239", "hostIPs": [{"ip": "10.0.133.239"}], "observedGeneration": 1, "phase": "Running", "podIP": "10.133.0.247", "podIPs": [{"ip": "10.133.0.247"}], "qosClass": "Burstable", "startTime": "2026-11-02T13:03:47Z"}} [pod/payload-processing-external-model-e2e-extmod-b-3a78b4-589cd7l2n/payload-processing] 2026-11-02T13:03:48.056344Z  INFO praxis_extproc::fips: installed rustls crypto provider provider="openssl" provider_fips=false kernel_fips=Some(false) fips_required=false [pod/payload-processing-external-model-e2e-extmod-b-3a78b4-589cd7l2n/payload-processing] 2026-11-02T13:03:48.063723Z  INFO praxis_ai_filters::routing::intelligent_route: intelligent_route: overlay snapshot initialized path=/etc/praxis/routing/routing-overlay.json contract_format=Envelope schema_version="1.0.0" accepted_revision="536beb84e492f28ffb1e3aa57748c557ecc2933b4d090f3f2c09d3251df06eac" serving_revision="536beb84e492f28ffb1e3aa57748c557ecc2933b4d090f3f2c09d3251df06eac" candidate_count=2 [pod/payload-processing-external-model-e2e-extmod-b-3a78b4-589cd7l2n/payload-processing] 2026-11-02T13:03:48.064056Z  INFO praxis_ai_filters::routing::overlay: intelligent_route: overlay file watcher started path=/etc/praxis/routing/routing-overlay.json debounce_ms=500 [pod/payload-processing-external-model-e2e-extmod-b-3a78b4-589cd7l2n/payload-processing] 2026-11-02T13:03:48.064706Z  INFO praxis_extproc: starting ExtProc server grpc=0.0.0.0:9004 health=0.0.0.0:50052 metrics=0.0.0.0:9090 filters=3 [pod/payload-processing-external-model-e2e-extmod-b-3a78b4-589cd7l2n/payload-processing] 2026-11-02T13:03:48.064733Z  INFO praxis_extproc: gRPC max decoding message size configured bytes=33554432 env="GRPC_MAX_RECV_MSG_SIZE" [pod/payload-processing-external-model-e2e-extmod-b-3a78b4-589cd7l2n/payload-processing] 2026-11-02T13:03:48.064739Z  INFO praxis_extproc::tls: generating self-signed TLS certificate [pod/payload-processing-external-model-e2e-extmod-b-3a78b4-589cd7l2n/payload-processing] 2026-11-02T13:03:48.064760Z  INFO praxis_extproc::health: health server listening address=0.0.0.0:50052 serving=true fips=false [pod/payload-processing-external-model-e2e-extmod-b-3a78b4-589cd7l2n/payload-processing] 2026-11-02T13:03:48.065486Z  INFO praxis_extproc::metrics: metrics server listening address=0.0.0.0:9090