tests/test_tenant_external_model_attachment.py::TestAttachmentLifecycleCleanup::test_detach_removes_only_that_attachment@tenant_ipp
{"namespace": "ai-tenant-e2e-extmod-b-3a78b4", "pod": "payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2", "status": {"conditions": [{"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:19:41Z", "observedGeneration": 1, "status": "True", "type": "PodReadyToStartContainers"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:19:40Z", "observedGeneration": 1, "status": "True", "type": "Initialized"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:19:52Z", "observedGeneration": 1, "status": "True", "type": "Ready"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:19:52Z", "observedGeneration": 1, "status": "True", "type": "ContainersReady"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:19:40Z", "observedGeneration": 1, "status": "True", "type": "PodScheduled"}], "containerStatuses": [{"allocatedResources": {"cpu": "50m", "memory": "64Mi"}, "containerID": "cri-o://8fc898b8c6b0e3b90309d3c2bf917b973eaa7f6af2f3df2e27c0a9803d070ca3", "image": "quay.io/opendatahub/odh-praxis-extproc:odh-stable", "imageID": "quay.io/opendatahub/odh-praxis-extproc@sha256:8b7084478cc266b2373bbfb04190ed3ca6fe759428663bddbab63b9c3ad8f130", "lastState": {}, "name": "payload-processing", "ready": true, "resources": {"limits": {"cpu": "500m", "memory": "256Mi"}, "requests": {"cpu": "50m", "memory": "64Mi"}}, "restartCount": 0, "started": true, "state": {"running": {"startedAt": "2026-11-02T12:19:41Z"}}, "user": {"linux": {"gid": 0, "supplementalGroups": [0, 1000850000], "uid": 1000850000}}, "volumeMounts": [{"mountPath": "/etc/praxis", "name": "plugins-config-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/routing", "name": "routing-overlay-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/credentials", "name": "provider-credentials", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", "name": "kube-api-access-mwtnc", "readOnly": true, "recursiveReadOnly": "Disabled"}]}], "hostIP": "10.0.135.24", "hostIPs": [{"ip": "10.0.135.24"}], "observedGeneration": 1, "phase": "Running", "podIP": "10.132.0.90", "podIPs": [{"ip": "10.132.0.90"}], "qosClass": "Burstable", "startTime": "2026-11-02T12:19:40Z"}}
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:19:41.098376Z[0m [32m INFO[0m [2mpraxis_extproc::fips[0m[2m:[0m installed rustls crypto provider [3mprovider[0m[2m=[0m"openssl" [3mprovider_fips[0m[2m=[0mfalse [3mkernel_fips[0m[2m=[0mSome(false) [3mfips_required[0m[2m=[0mfalse
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:19:41.106005Z[0m [32m INFO[0m [2mpraxis_ai_filters::routing::intelligent_route[0m[2m:[0m intelligent_route: overlay snapshot initialized [3mpath[0m[2m=[0m/etc/praxis/routing/routing-overlay.json [3mcontract_format[0m[2m=[0mEnvelope [3mschema_version[0m[2m=[0m"1.0.0" [3maccepted_revision[0m[2m=[0m"1eb19e4588c2422b748462adc08cd4905bb95310f9504e8f84789c7f7e1d11ee" [3mserving_revision[0m[2m=[0m"1eb19e4588c2422b748462adc08cd4905bb95310f9504e8f84789c7f7e1d11ee" [3mcandidate_count[0m[2m=[0m3
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:19:41.106426Z[0m [32m INFO[0m [2mpraxis_ai_filters::routing::overlay[0m[2m:[0m intelligent_route: overlay file watcher started [3mpath[0m[2m=[0m/etc/praxis/routing/routing-overlay.json [3mdebounce_ms[0m[2m=[0m500
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:19:41.107683Z[0m [32m INFO[0m [2mpraxis_extproc[0m[2m:[0m starting ExtProc server [3mgrpc[0m[2m=[0m0.0.0.0:9004 [3mhealth[0m[2m=[0m0.0.0.0:50052 [3mmetrics[0m[2m=[0m0.0.0.0:9090 [3mfilters[0m[2m=[0m3
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:19:41.107718Z[0m [32m INFO[0m [2mpraxis_extproc[0m[2m:[0m gRPC max decoding message size configured [3mbytes[0m[2m=[0m33554432 [3menv[0m[2m=[0m"GRPC_MAX_RECV_MSG_SIZE"
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:19:41.107726Z[0m [32m INFO[0m [2mpraxis_extproc::tls[0m[2m:[0m generating self-signed TLS certificate
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:19:41.107748Z[0m [32m INFO[0m [2mpraxis_extproc::health[0m[2m:[0m health server listening [3maddress[0m[2m=[0m0.0.0.0:50052 [3mserving[0m[2m=[0mtrue [3mfips[0m[2m=[0mfalse
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:19:41.108559Z[0m [32m INFO[0m [2mpraxis_extproc::metrics[0m[2m:[0m metrics server listening [3maddress[0m[2m=[0m0.0.0.0:9090
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:20:33.482002Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:20:38.908863Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:20:44.182433Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:20:50.259404Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:20:55.623476Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:21:01.830939Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:21:07.079489Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:21:12.504185Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:21:17.754457Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:21:23.008398Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:21:28.262682Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:21:33.534398Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:21:38.788691Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:21:44.228611Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:21:49.510372Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:21:54.754785Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:22:00.105040Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:22:05.354512Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:22:10.670403Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:22:16.076522Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:22:22.814175Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:22:28.070128Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:22:33.344809Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:22:39.664036Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:22:44.932013Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:22:50.372446Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:22:55.697975Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:23:00.966139Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:23:06.223722Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:23:11.512059Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:23:16.790087Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:23:22.248819Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:23:27.534201Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:23:32.822863Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:23:38.131814Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-8696z46f2/payload-processing] [2m2026-11-02T12:23:43.397350Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503

{"namespace": "models-as-a-service", "pod": "payload-processing-external-model-749cdcd88d-dm2dx", "status": {"conditions": [{"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:23:21Z", "observedGeneration": 1, "status": "True", "type": "PodReadyToStartContainers"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:23:19Z", "observedGeneration": 1, "status": "True", "type": "Initialized"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:23:19Z", "message": "containers with unready status: [payload-processing]", "observedGeneration": 1, "reason": "ContainersNotReady", "status": "False", "type": "Ready"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:23:19Z", "message": "containers with unready status: [payload-processing]", "observedGeneration": 1, "reason": "ContainersNotReady", "status": "False", "type": "ContainersReady"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:23:19Z", "observedGeneration": 1, "status": "True", "type": "PodScheduled"}], "containerStatuses": [{"allocatedResources": {"cpu": "50m", "memory": "64Mi"}, "containerID": "cri-o://c5e10721fcc1714fc4c8acad4361fe5161bc349ccc20631b50e44a4e1db8ee02", "image": "quay.io/opendatahub/odh-praxis-extproc:odh-stable", "imageID": "quay.io/opendatahub/odh-praxis-extproc@sha256:8b7084478cc266b2373bbfb04190ed3ca6fe759428663bddbab63b9c3ad8f130", "lastState": {}, "name": "payload-processing", "ready": false, "resources": {"limits": {"cpu": "500m", "memory": "256Mi"}, "requests": {"cpu": "50m", "memory": "64Mi"}}, "restartCount": 0, "started": true, "state": {"running": {"startedAt": "2026-11-02T12:23:20Z"}}, "user": {"linux": {"gid": 0, "supplementalGroups": [0, 1000700000], "uid": 1000700000}}, "volumeMounts": [{"mountPath": "/etc/praxis", "name": "plugins-config-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/routing", "name": "routing-overlay-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/credentials", "name": "provider-credentials", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", "name": "kube-api-access-rpdv7", "readOnly": true, "recursiveReadOnly": "Disabled"}]}], "hostIP": "10.0.133.239", "hostIPs": [{"ip": "10.0.133.239"}], "observedGeneration": 1, "phase": "Running", "podIP": "10.133.0.151", "podIPs": [{"ip": "10.133.0.151"}], "qosClass": "Burstable", "startTime": "2026-11-02T12:23:19Z"}}
[pod/payload-processing-external-model-749cdcd88d-dm2dx/payload-processing] [2m2026-11-02T12:23:20.568680Z[0m [32m INFO[0m [2mpraxis_extproc::fips[0m[2m:[0m installed rustls crypto provider [3mprovider[0m[2m=[0m"openssl" [3mprovider_fips[0m[2m=[0mfalse [3mkernel_fips[0m[2m=[0mSome(false) [3mfips_required[0m[2m=[0mfalse
[pod/payload-processing-external-model-749cdcd88d-dm2dx/payload-processing] [2m2026-11-02T12:23:20.576205Z[0m [32m INFO[0m [2mpraxis_ai_filters::routing::intelligent_route[0m[2m:[0m intelligent_route: overlay snapshot initialized [3mpath[0m[2m=[0m/etc/praxis/routing/routing-overlay.json [3mcontract_format[0m[2m=[0mEnvelope [3mschema_version[0m[2m=[0m"1.0.0" [3maccepted_revision[0m[2m=[0m"3a705aa4795cb19dabe89d9fb4249659521029e646ad125199b1e3c6343c1ffc" [3mserving_revision[0m[2m=[0m"3a705aa4795cb19dabe89d9fb4249659521029e646ad125199b1e3c6343c1ffc" [3mcandidate_count[0m[2m=[0m1
[pod/payload-processing-external-model-749cdcd88d-dm2dx/payload-processing] [2m2026-11-02T12:23:20.576511Z[0m [32m INFO[0m [2mpraxis_ai_filters::routing::overlay[0m[2m:[0m intelligent_route: overlay file watcher started [3mpath[0m[2m=[0m/etc/praxis/routing/routing-overlay.json [3mdebounce_ms[0m[2m=[0m500
[pod/payload-processing-external-model-749cdcd88d-dm2dx/payload-processing] [2m2026-11-02T12:23:20.576607Z[0m [32m INFO[0m [2mpraxis_ai_filters::routing::overlay[0m[2m:[0m intelligent_route: overlay file watcher shutting down
[pod/payload-processing-external-model-749cdcd88d-dm2dx/payload-processing] [2m2026-11-02T12:23:20.581679Z[0m [31mERROR[0m [2mpraxis_extproc[0m[2m:[0m filter pipeline build failed; reporting NotServing [3merror[0m[2m=[0mpipeline: credential_inject: cannot read file '/etc/praxis/credentials/agc-cred-6278d2f81b78-2c0419d93bd2/api-key' for 'agc-cred-6278d2f81b78/models-as-a-service/api-key': No such file or directory (os error 2) [3mhealth[0m[2m=[0m0.0.0.0:50052
[pod/payload-processing-external-model-749cdcd88d-dm2dx/payload-processing] [2m2026-11-02T12:23:20.581734Z[0m [32m INFO[0m [2mpraxis_extproc::health[0m[2m:[0m health server listening [3maddress[0m[2m=[0m0.0.0.0:50052 [3mserving[0m[2m=[0mfalse [3mfips[0m[2m=[0mfalse
[pod/payload-processing-external-model-749cdcd88d-dm2dx/payload-processing] [2m2026-11-02T12:23:20.582484Z[0m [32m INFO[0m [2mpraxis_extproc::metrics[0m[2m:[0m metrics server listening [3maddress[0m[2m=[0m0.0.0.0:9090

{"namespace": "models-as-a-service", "pod": "payload-processing-external-model-866d944846-nnbj2", "status": {"conditions": [{"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:22:49Z", "observedGeneration": 1, "status": "True", "type": "PodReadyToStartContainers"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:22:48Z", "observedGeneration": 1, "status": "True", "type": "Initialized"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:23:00Z", "observedGeneration": 1, "status": "True", "type": "Ready"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:23:00Z", "observedGeneration": 1, "status": "True", "type": "ContainersReady"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:22:48Z", "observedGeneration": 1, "status": "True", "type": "PodScheduled"}], "containerStatuses": [{"allocatedResources": {"cpu": "50m", "memory": "64Mi"}, "containerID": "cri-o://1c843bcd4cc5a173c66216d8892e3db90897feb8d66956a13cba2be0ec2a01c2", "image": "quay.io/opendatahub/odh-praxis-extproc:odh-stable", "imageID": "quay.io/opendatahub/odh-praxis-extproc@sha256:8b7084478cc266b2373bbfb04190ed3ca6fe759428663bddbab63b9c3ad8f130", "lastState": {}, "name": "payload-processing", "ready": true, "resources": {"limits": {"cpu": "500m", "memory": "256Mi"}, "requests": {"cpu": "50m", "memory": "64Mi"}}, "restartCount": 0, "started": true, "state": {"running": {"startedAt": "2026-11-02T12:22:49Z"}}, "user": {"linux": {"gid": 0, "supplementalGroups": [0, 1000700000], "uid": 1000700000}}, "volumeMounts": [{"mountPath": "/etc/praxis", "name": "plugins-config-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/routing", "name": "routing-overlay-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/credentials", "name": "provider-credentials", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", "name": "kube-api-access-75bw4", "readOnly": true, "recursiveReadOnly": "Disabled"}]}], "hostIP": "10.0.133.239", "hostIPs": [{"ip": "10.0.133.239"}], "observedGeneration": 1, "phase": "Running", "podIP": "10.133.0.145", "podIPs": [{"ip": "10.133.0.145"}], "qosClass": "Burstable", "startTime": "2026-11-02T12:22:48Z"}}
[pod/payload-processing-external-model-866d944846-nnbj2/payload-processing] [2m2026-11-02T12:22:49.217380Z[0m [32m INFO[0m [2mpraxis_extproc::fips[0m[2m:[0m installed rustls crypto provider [3mprovider[0m[2m=[0m"openssl" [3mprovider_fips[0m[2m=[0mfalse [3mkernel_fips[0m[2m=[0mSome(false) [3mfips_required[0m[2m=[0mfalse
[pod/payload-processing-external-model-866d944846-nnbj2/payload-processing] [2m2026-11-02T12:22:49.225278Z[0m [32m INFO[0m [2mpraxis_ai_filters::routing::intelligent_route[0m[2m:[0m intelligent_route: overlay snapshot initialized [3mpath[0m[2m=[0m/etc/praxis/routing/routing-overlay.json [3mcontract_format[0m[2m=[0mEnvelope [3mschema_version[0m[2m=[0m"1.0.0" [3maccepted_revision[0m[2m=[0m"6379dd9bb2090689ff9e61719cd1a48395cd778ab29f4fef77bab5dd2d031cd5" [3mserving_revision[0m[2m=[0m"6379dd9bb2090689ff9e61719cd1a48395cd778ab29f4fef77bab5dd2d031cd5" [3mcandidate_count[0m[2m=[0m2
[pod/payload-processing-external-model-866d944846-nnbj2/payload-processing] [2m2026-11-02T12:22:49.225635Z[0m [32m INFO[0m [2mpraxis_ai_filters::routing::overlay[0m[2m:[0m intelligent_route: overlay file watcher started [3mpath[0m[2m=[0m/etc/praxis/routing/routing-overlay.json [3mdebounce_ms[0m[2m=[0m500
[pod/payload-processing-external-model-866d944846-nnbj2/payload-processing] [2m2026-11-02T12:22:49.226609Z[0m [32m INFO[0m [2mpraxis_extproc[0m[2m:[0m starting ExtProc server [3mgrpc[0m[2m=[0m0.0.0.0:9004 [3mhealth[0m[2m=[0m0.0.0.0:50052 [3mmetrics[0m[2m=[0m0.0.0.0:9090 [3mfilters[0m[2m=[0m3
[pod/payload-processing-external-model-866d944846-nnbj2/payload-processing] [2m2026-11-02T12:22:49.226646Z[0m [32m INFO[0m [2mpraxis_extproc[0m[2m:[0m gRPC max decoding message size configured [3mbytes[0m[2m=[0m33554432 [3menv[0m[2m=[0m"GRPC_MAX_RECV_MSG_SIZE"
[pod/payload-processing-external-model-866d944846-nnbj2/payload-processing] [2m2026-11-02T12:22:49.226654Z[0m [32m INFO[0m [2mpraxis_extproc::tls[0m[2m:[0m generating self-signed TLS certificate
[pod/payload-processing-external-model-866d944846-nnbj2/payload-processing] [2m2026-11-02T12:22:49.226663Z[0m [32m INFO[0m [2mpraxis_extproc::health[0m[2m:[0m health server listening [3maddress[0m[2m=[0m0.0.0.0:50052 [3mserving[0m[2m=[0mtrue [3mfips[0m[2m=[0mfalse
[pod/payload-processing-external-model-866d944846-nnbj2/payload-processing] [2m2026-11-02T12:22:49.227478Z[0m [32m INFO[0m [2mpraxis_extproc::metrics[0m[2m:[0m metrics server listening [3maddress[0m[2m=[0m0.0.0.0:9090