tests/test_tenant_external_model_shared.py::TestTenantQuotaIsolation::test_quota_exhaustion_isolated_per_tenant@tenant_ipp {"namespace": "ai-tenant-e2e-extmod-a-6ac03f", "pod": "payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w", "status": {"conditions": [{"lastProbeTime": null, "lastTransitionTime": "2026-11-02T13:07:59Z", "observedGeneration": 1, "status": "True", "type": "PodReadyToStartContainers"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T13:07:58Z", "observedGeneration": 1, "status": "True", "type": "Initialized"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T13:08:10Z", "observedGeneration": 1, "status": "True", "type": "Ready"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T13:08:10Z", "observedGeneration": 1, "status": "True", "type": "ContainersReady"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T13:07:58Z", "observedGeneration": 1, "status": "True", "type": "PodScheduled"}], "containerStatuses": [{"allocatedResources": {"cpu": "50m", "memory": "64Mi"}, "containerID": "cri-o://000b755a77737abf603701afea6768f6b80c36c01ea607333d61a81dc50f8198", "image": "quay.io/opendatahub/odh-praxis-extproc:odh-stable", "imageID": "quay.io/opendatahub/odh-praxis-extproc@sha256:8b7084478cc266b2373bbfb04190ed3ca6fe759428663bddbab63b9c3ad8f130", "lastState": {}, "name": "payload-processing", "ready": true, "resources": {"limits": {"cpu": "500m", "memory": "256Mi"}, "requests": {"cpu": "50m", "memory": "64Mi"}}, "restartCount": 0, "started": true, "state": {"running": {"startedAt": "2026-11-02T13:07:59Z"}}, "user": {"linux": {"gid": 0, "supplementalGroups": [0, 1000840000], "uid": 1000840000}}, "volumeMounts": [{"mountPath": "/etc/praxis", "name": "plugins-config-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/routing", "name": "routing-overlay-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/credentials", "name": "provider-credentials", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", "name": "kube-api-access-x2hbx", "readOnly": true, "recursiveReadOnly": "Disabled"}]}], "hostIP": "10.0.133.239", "hostIPs": [{"ip": "10.0.133.239"}], "observedGeneration": 1, "phase": "Running", "podIP": "10.133.0.252", "podIPs": [{"ip": "10.133.0.252"}], "qosClass": "Burstable", "startTime": "2026-11-02T13:07:58Z"}} [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:07:59.358905Z  INFO praxis_extproc::fips: installed rustls crypto provider provider="openssl" provider_fips=false kernel_fips=Some(false) fips_required=false [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:07:59.366343Z  INFO praxis_ai_filters::routing::intelligent_route: intelligent_route: overlay snapshot initialized path=/etc/praxis/routing/routing-overlay.json contract_format=Envelope schema_version="1.0.0" accepted_revision="2883668dd344df8f3b98a9d2f47b50ccf208cad3d9b5a689751cc3d8276f372c" serving_revision="2883668dd344df8f3b98a9d2f47b50ccf208cad3d9b5a689751cc3d8276f372c" candidate_count=4 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:07:59.366671Z  INFO praxis_ai_filters::routing::overlay: intelligent_route: overlay file watcher started path=/etc/praxis/routing/routing-overlay.json debounce_ms=500 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:07:59.367927Z  INFO praxis_extproc: starting ExtProc server grpc=0.0.0.0:9004 health=0.0.0.0:50052 metrics=0.0.0.0:9090 filters=3 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:07:59.367960Z  INFO praxis_extproc: gRPC max decoding message size configured bytes=33554432 env="GRPC_MAX_RECV_MSG_SIZE" [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:07:59.367967Z  INFO praxis_extproc::tls: generating self-signed TLS certificate [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:07:59.367994Z  INFO praxis_extproc::health: health server listening address=0.0.0.0:50052 serving=true fips=false [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:07:59.368710Z  INFO praxis_extproc::metrics: metrics server listening address=0.0.0.0:9090 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:08:34.886857Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:08:40.302154Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:08:45.567702Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:08:50.827576Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:08:56.084287Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:09:01.344862Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:09:06.602329Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:09:12.023884Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:09:17.277589Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:09:22.534214Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:09:27.788759Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:09:33.038105Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:09:38.288341Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:09:43.705615Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:09:48.961277Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:09:54.213679Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:09:59.473105Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:10:04.738847Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:10:09.992145Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:10:15.402269Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:10:20.652274Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:10:25.910069Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:10:31.163791Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:10:36.425476Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:10:41.684290Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:10:47.102207Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:10:52.359542Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:10:57.613995Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:11:02.867778Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:11:08.128454Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:11:13.382405Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:11:18.802050Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:11:24.060702Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:11:29.319682Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:11:34.585657Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 [pod/payload-processing-external-model-e2e-extmod-a-6ac03f-c47dxfx4w/payload-processing] 2026-11-02T13:11:39.846015Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503 {"namespace": "ai-tenant-e2e-extmod-b-3a78b4", "pod": "payload-processing-external-model-e2e-extmod-b-3a78b4-b694542hx", "status": {"conditions": [{"lastProbeTime": null, "lastTransitionTime": "2026-11-02T13:07:56Z", "observedGeneration": 1, "status": "True", "type": "PodReadyToStartContainers"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T13:07:55Z", "observedGeneration": 1, "status": "True", "type": "Initialized"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T13:08:07Z", "observedGeneration": 1, "status": "True", "type": "Ready"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T13:08:07Z", "observedGeneration": 1, "status": "True", "type": "ContainersReady"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T13:07:55Z", "observedGeneration": 1, "status": "True", "type": "PodScheduled"}], "containerStatuses": [{"allocatedResources": {"cpu": "50m", "memory": "64Mi"}, "containerID": "cri-o://4ba05b6da2b4cce4eb7333380e8f1d849bce6d26671cfe5977e600595c6273a9", "image": "quay.io/opendatahub/odh-praxis-extproc:odh-stable", "imageID": "quay.io/opendatahub/odh-praxis-extproc@sha256:8b7084478cc266b2373bbfb04190ed3ca6fe759428663bddbab63b9c3ad8f130", "lastState": {}, "name": "payload-processing", "ready": true, "resources": {"limits": {"cpu": "500m", "memory": "256Mi"}, "requests": {"cpu": "50m", "memory": "64Mi"}}, "restartCount": 0, "started": true, "state": {"running": {"startedAt": "2026-11-02T13:07:56Z"}}, "user": {"linux": {"gid": 0, "supplementalGroups": [0, 1000850000], "uid": 1000850000}}, "volumeMounts": [{"mountPath": "/etc/praxis", "name": "plugins-config-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/routing", "name": "routing-overlay-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/credentials", "name": "provider-credentials", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", "name": "kube-api-access-fp7xc", "readOnly": true, "recursiveReadOnly": "Disabled"}]}], "hostIP": "10.0.133.239", "hostIPs": [{"ip": "10.0.133.239"}], "observedGeneration": 1, "phase": "Running", "podIP": "10.133.0.251", "podIPs": [{"ip": "10.133.0.251"}], "qosClass": "Burstable", "startTime": "2026-11-02T13:07:55Z"}} [pod/payload-processing-external-model-e2e-extmod-b-3a78b4-b694542hx/payload-processing] 2026-11-02T13:07:56.475572Z  INFO praxis_extproc::fips: installed rustls crypto provider provider="openssl" provider_fips=false kernel_fips=Some(false) fips_required=false [pod/payload-processing-external-model-e2e-extmod-b-3a78b4-b694542hx/payload-processing] 2026-11-02T13:07:56.482943Z  INFO praxis_ai_filters::routing::intelligent_route: intelligent_route: overlay snapshot initialized path=/etc/praxis/routing/routing-overlay.json contract_format=Envelope schema_version="1.0.0" accepted_revision="d84b0ba93f44446a326677b644068216a0f0acf0039114f1385b9e74e6d7e6ee" serving_revision="d84b0ba93f44446a326677b644068216a0f0acf0039114f1385b9e74e6d7e6ee" candidate_count=4 [pod/payload-processing-external-model-e2e-extmod-b-3a78b4-b694542hx/payload-processing] 2026-11-02T13:07:56.483266Z  INFO praxis_ai_filters::routing::overlay: intelligent_route: overlay file watcher started path=/etc/praxis/routing/routing-overlay.json debounce_ms=500 [pod/payload-processing-external-model-e2e-extmod-b-3a78b4-b694542hx/payload-processing] 2026-11-02T13:07:56.484517Z  INFO praxis_extproc: starting ExtProc server grpc=0.0.0.0:9004 health=0.0.0.0:50052 metrics=0.0.0.0:9090 filters=3 [pod/payload-processing-external-model-e2e-extmod-b-3a78b4-b694542hx/payload-processing] 2026-11-02T13:07:56.484543Z  INFO praxis_extproc: gRPC max decoding message size configured bytes=33554432 env="GRPC_MAX_RECV_MSG_SIZE" [pod/payload-processing-external-model-e2e-extmod-b-3a78b4-b694542hx/payload-processing] 2026-11-02T13:07:56.484548Z  INFO praxis_extproc::tls: generating self-signed TLS certificate [pod/payload-processing-external-model-e2e-extmod-b-3a78b4-b694542hx/payload-processing] 2026-11-02T13:07:56.484570Z  INFO praxis_extproc::health: health server listening address=0.0.0.0:50052 serving=true fips=false [pod/payload-processing-external-model-e2e-extmod-b-3a78b4-b694542hx/payload-processing] 2026-11-02T13:07:56.485318Z  INFO praxis_extproc::metrics: metrics server listening address=0.0.0.0:9090