tests/test_tenant_external_model_attachment.py::TestAttachmentLifecycleCleanup::test_detach_removes_only_that_attachment@tenant_ipp
{"namespace": "ai-tenant-e2e-extmod-b-8c9b0f", "pod": "payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx", "status": {"conditions": [{"lastProbeTime": null, "lastTransitionTime": "2026-11-03T17:37:46Z", "observedGeneration": 1, "status": "True", "type": "PodReadyToStartContainers"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-03T17:37:45Z", "observedGeneration": 1, "status": "True", "type": "Initialized"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-03T17:37:57Z", "observedGeneration": 1, "status": "True", "type": "Ready"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-03T17:37:57Z", "observedGeneration": 1, "status": "True", "type": "ContainersReady"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-03T17:37:45Z", "observedGeneration": 1, "status": "True", "type": "PodScheduled"}], "containerStatuses": [{"allocatedResources": {"cpu": "50m", "memory": "64Mi"}, "containerID": "cri-o://43ab9a3dcb4dcc1a64aec263f036f2666300e0a1966e6e06866ca2fc63544e0f", "image": "quay.io/opendatahub/odh-praxis-extproc:odh-stable", "imageID": "quay.io/opendatahub/odh-praxis-extproc@sha256:8b7084478cc266b2373bbfb04190ed3ca6fe759428663bddbab63b9c3ad8f130", "lastState": {}, "name": "payload-processing", "ready": true, "resources": {"limits": {"cpu": "500m", "memory": "256Mi"}, "requests": {"cpu": "50m", "memory": "64Mi"}}, "restartCount": 0, "started": true, "state": {"running": {"startedAt": "2026-11-03T17:37:46Z"}}, "user": {"linux": {"gid": 0, "supplementalGroups": [0, 1000860000], "uid": 1000860000}}, "volumeMounts": [{"mountPath": "/etc/praxis", "name": "plugins-config-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/routing", "name": "routing-overlay-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/credentials", "name": "provider-credentials", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", "name": "kube-api-access-xw5hq", "readOnly": true, "recursiveReadOnly": "Disabled"}]}], "hostIP": "10.0.143.117", "hostIPs": [{"ip": "10.0.143.117"}], "observedGeneration": 1, "phase": "Running", "podIP": "10.133.0.121", "podIPs": [{"ip": "10.133.0.121"}], "qosClass": "Burstable", "startTime": "2026-11-03T17:37:45Z"}}
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:37:46.057630Z[0m [32m INFO[0m [2mpraxis_extproc::fips[0m[2m:[0m installed rustls crypto provider [3mprovider[0m[2m=[0m"openssl" [3mprovider_fips[0m[2m=[0mfalse [3mkernel_fips[0m[2m=[0mSome(false) [3mfips_required[0m[2m=[0mfalse
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:37:46.064945Z[0m [32m INFO[0m [2mpraxis_ai_filters::routing::intelligent_route[0m[2m:[0m intelligent_route: overlay snapshot initialized [3mpath[0m[2m=[0m/etc/praxis/routing/routing-overlay.json [3mcontract_format[0m[2m=[0mEnvelope [3mschema_version[0m[2m=[0m"1.0.0" [3maccepted_revision[0m[2m=[0m"17d7725ee95ff9a721a9b997451539de3d9e2297524d6a12014b7ea78f6adf84" [3mserving_revision[0m[2m=[0m"17d7725ee95ff9a721a9b997451539de3d9e2297524d6a12014b7ea78f6adf84" [3mcandidate_count[0m[2m=[0m3
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:37:46.065293Z[0m [32m INFO[0m [2mpraxis_ai_filters::routing::overlay[0m[2m:[0m intelligent_route: overlay file watcher started [3mpath[0m[2m=[0m/etc/praxis/routing/routing-overlay.json [3mdebounce_ms[0m[2m=[0m500
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:37:46.066292Z[0m [32m INFO[0m [2mpraxis_extproc[0m[2m:[0m starting ExtProc server [3mgrpc[0m[2m=[0m0.0.0.0:9004 [3mhealth[0m[2m=[0m0.0.0.0:50052 [3mmetrics[0m[2m=[0m0.0.0.0:9090 [3mfilters[0m[2m=[0m3
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:37:46.066317Z[0m [32m INFO[0m [2mpraxis_extproc[0m[2m:[0m gRPC max decoding message size configured [3mbytes[0m[2m=[0m33554432 [3menv[0m[2m=[0m"GRPC_MAX_RECV_MSG_SIZE"
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:37:46.066322Z[0m [32m INFO[0m [2mpraxis_extproc::tls[0m[2m:[0m generating self-signed TLS certificate
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:37:46.066342Z[0m [32m INFO[0m [2mpraxis_extproc::health[0m[2m:[0m health server listening [3maddress[0m[2m=[0m0.0.0.0:50052 [3mserving[0m[2m=[0mtrue [3mfips[0m[2m=[0mfalse
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:37:46.067068Z[0m [32m INFO[0m [2mpraxis_extproc::metrics[0m[2m:[0m metrics server listening [3maddress[0m[2m=[0m0.0.0.0:9090
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:38:42.850292Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:38:48.405601Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:38:53.727623Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:38:59.069178Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:39:04.382738Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:39:09.673460Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:39:15.033465Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:39:20.526551Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:39:25.842826Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:39:38.343409Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:39:43.648580Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:39:49.045096Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:39:54.545486Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:39:59.857102Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:40:05.154474Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:40:10.462177Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:40:15.772067Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:40:21.087035Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:40:26.613428Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:40:31.906176Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:40:37.212473Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:40:42.612521Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:40:47.920895Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:40:53.233490Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:40:58.823502Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:41:04.113335Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:41:09.412436Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:41:14.761516Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:41:20.101813Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:41:25.423685Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:41:30.900311Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:41:36.286424Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:41:41.575820Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:41:46.873588Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-8c9b0f-fc8f25xdx/payload-processing] [2m2026-11-03T17:41:52.195099Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503

{"namespace": "models-as-a-service", "pod": "payload-processing-external-model-67bfdd8c49-4rgpl", "status": {"conditions": [{"lastProbeTime": null, "lastTransitionTime": "2026-11-03T17:41:29Z", "observedGeneration": 1, "status": "True", "type": "PodReadyToStartContainers"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-03T17:41:25Z", "observedGeneration": 1, "status": "True", "type": "Initialized"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-03T17:41:25Z", "message": "containers with unready status: [payload-processing]", "observedGeneration": 1, "reason": "ContainersNotReady", "status": "False", "type": "Ready"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-03T17:41:25Z", "message": "containers with unready status: [payload-processing]", "observedGeneration": 1, "reason": "ContainersNotReady", "status": "False", "type": "ContainersReady"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-03T17:41:25Z", "observedGeneration": 1, "status": "True", "type": "PodScheduled"}], "containerStatuses": [{"allocatedResources": {"cpu": "50m", "memory": "64Mi"}, "containerID": "cri-o://dafb2d78d8bb1fcf26dc4f64f57d151133e52ba078867c75a7e51c6b258e8e52", "image": "quay.io/opendatahub/odh-praxis-extproc:odh-stable", "imageID": "quay.io/opendatahub/odh-praxis-extproc@sha256:8b7084478cc266b2373bbfb04190ed3ca6fe759428663bddbab63b9c3ad8f130", "lastState": {}, "name": "payload-processing", "ready": false, "resources": {"limits": {"cpu": "500m", "memory": "256Mi"}, "requests": {"cpu": "50m", "memory": "64Mi"}}, "restartCount": 0, "started": true, "state": {"running": {"startedAt": "2026-11-03T17:41:28Z"}}, "user": {"linux": {"gid": 0, "supplementalGroups": [0, 1000710000], "uid": 1000710000}}, "volumeMounts": [{"mountPath": "/etc/praxis", "name": "plugins-config-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/routing", "name": "routing-overlay-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/credentials", "name": "provider-credentials", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", "name": "kube-api-access-k8sht", "readOnly": true, "recursiveReadOnly": "Disabled"}]}], "hostIP": "10.0.129.76", "hostIPs": [{"ip": "10.0.129.76"}], "observedGeneration": 1, "phase": "Running", "podIP": "10.134.0.69", "podIPs": [{"ip": "10.134.0.69"}], "qosClass": "Burstable", "startTime": "2026-11-03T17:41:25Z"}}
[pod/payload-processing-external-model-67bfdd8c49-4rgpl/payload-processing] [2m2026-11-03T17:41:28.920368Z[0m [32m INFO[0m [2mpraxis_extproc::fips[0m[2m:[0m installed rustls crypto provider [3mprovider[0m[2m=[0m"openssl" [3mprovider_fips[0m[2m=[0mfalse [3mkernel_fips[0m[2m=[0mSome(false) [3mfips_required[0m[2m=[0mfalse
[pod/payload-processing-external-model-67bfdd8c49-4rgpl/payload-processing] [2m2026-11-03T17:41:28.928050Z[0m [32m INFO[0m [2mpraxis_ai_filters::routing::intelligent_route[0m[2m:[0m intelligent_route: overlay snapshot initialized [3mpath[0m[2m=[0m/etc/praxis/routing/routing-overlay.json [3mcontract_format[0m[2m=[0mEnvelope [3mschema_version[0m[2m=[0m"1.0.0" [3maccepted_revision[0m[2m=[0m"d81c9d5a10e143fbd71af1749f497c4c664331972a75e9653c3d7eac13f69cd0" [3mserving_revision[0m[2m=[0m"d81c9d5a10e143fbd71af1749f497c4c664331972a75e9653c3d7eac13f69cd0" [3mcandidate_count[0m[2m=[0m1
[pod/payload-processing-external-model-67bfdd8c49-4rgpl/payload-processing] [2m2026-11-03T17:41:28.928377Z[0m [32m INFO[0m [2mpraxis_ai_filters::routing::overlay[0m[2m:[0m intelligent_route: overlay file watcher started [3mpath[0m[2m=[0m/etc/praxis/routing/routing-overlay.json [3mdebounce_ms[0m[2m=[0m500
[pod/payload-processing-external-model-67bfdd8c49-4rgpl/payload-processing] [2m2026-11-03T17:41:28.928466Z[0m [32m INFO[0m [2mpraxis_ai_filters::routing::overlay[0m[2m:[0m intelligent_route: overlay file watcher shutting down
[pod/payload-processing-external-model-67bfdd8c49-4rgpl/payload-processing] [2m2026-11-03T17:41:28.933538Z[0m [31mERROR[0m [2mpraxis_extproc[0m[2m:[0m filter pipeline build failed; reporting NotServing [3merror[0m[2m=[0mpipeline: credential_inject: cannot read file '/etc/praxis/credentials/agc-cred-adbf463d8a89-4d66c0fb3e6b/api-key' for 'agc-cred-adbf463d8a89/models-as-a-service/api-key': No such file or directory (os error 2) [3mhealth[0m[2m=[0m0.0.0.0:50052
[pod/payload-processing-external-model-67bfdd8c49-4rgpl/payload-processing] [2m2026-11-03T17:41:28.933599Z[0m [32m INFO[0m [2mpraxis_extproc::health[0m[2m:[0m health server listening [3maddress[0m[2m=[0m0.0.0.0:50052 [3mserving[0m[2m=[0mfalse [3mfips[0m[2m=[0mfalse
[pod/payload-processing-external-model-67bfdd8c49-4rgpl/payload-processing] [2m2026-11-03T17:41:28.934435Z[0m [32m INFO[0m [2mpraxis_extproc::metrics[0m[2m:[0m metrics server listening [3maddress[0m[2m=[0m0.0.0.0:9090

{"namespace": "models-as-a-service", "pod": "payload-processing-external-model-8657f8dff8-j9qn4", "status": {"conditions": [{"lastProbeTime": null, "lastTransitionTime": "2026-11-03T17:40:57Z", "observedGeneration": 1, "status": "True", "type": "PodReadyToStartContainers"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-03T17:40:55Z", "observedGeneration": 1, "status": "True", "type": "Initialized"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-03T17:41:08Z", "observedGeneration": 1, "status": "True", "type": "Ready"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-03T17:41:08Z", "observedGeneration": 1, "status": "True", "type": "ContainersReady"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-03T17:40:55Z", "observedGeneration": 1, "status": "True", "type": "PodScheduled"}], "containerStatuses": [{"allocatedResources": {"cpu": "50m", "memory": "64Mi"}, "containerID": "cri-o://a2b0250db61dfffcb5a5f13bb0d0a1059e7eae71c38951e5ca9eecb5e88fac97", "image": "quay.io/opendatahub/odh-praxis-extproc:odh-stable", "imageID": "quay.io/opendatahub/odh-praxis-extproc@sha256:8b7084478cc266b2373bbfb04190ed3ca6fe759428663bddbab63b9c3ad8f130", "lastState": {}, "name": "payload-processing", "ready": true, "resources": {"limits": {"cpu": "500m", "memory": "256Mi"}, "requests": {"cpu": "50m", "memory": "64Mi"}}, "restartCount": 0, "started": true, "state": {"running": {"startedAt": "2026-11-03T17:40:56Z"}}, "user": {"linux": {"gid": 0, "supplementalGroups": [0, 1000710000], "uid": 1000710000}}, "volumeMounts": [{"mountPath": "/etc/praxis", "name": "plugins-config-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/routing", "name": "routing-overlay-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/credentials", "name": "provider-credentials", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", "name": "kube-api-access-82fjx", "readOnly": true, "recursiveReadOnly": "Disabled"}]}], "hostIP": "10.0.143.117", "hostIPs": [{"ip": "10.0.143.117"}], "observedGeneration": 1, "phase": "Running", "podIP": "10.133.0.144", "podIPs": [{"ip": "10.133.0.144"}], "qosClass": "Burstable", "startTime": "2026-11-03T17:40:55Z"}}
[pod/payload-processing-external-model-8657f8dff8-j9qn4/payload-processing] [2m2026-11-03T17:40:56.576910Z[0m [32m INFO[0m [2mpraxis_extproc::fips[0m[2m:[0m installed rustls crypto provider [3mprovider[0m[2m=[0m"openssl" [3mprovider_fips[0m[2m=[0mfalse [3mkernel_fips[0m[2m=[0mSome(false) [3mfips_required[0m[2m=[0mfalse
[pod/payload-processing-external-model-8657f8dff8-j9qn4/payload-processing] [2m2026-11-03T17:40:56.584486Z[0m [32m INFO[0m [2mpraxis_ai_filters::routing::intelligent_route[0m[2m:[0m intelligent_route: overlay snapshot initialized [3mpath[0m[2m=[0m/etc/praxis/routing/routing-overlay.json [3mcontract_format[0m[2m=[0mEnvelope [3mschema_version[0m[2m=[0m"1.0.0" [3maccepted_revision[0m[2m=[0m"a252bff9514bec1ed3977d5a42b02c2f10b24efe344f33fa596d9075feb57218" [3mserving_revision[0m[2m=[0m"a252bff9514bec1ed3977d5a42b02c2f10b24efe344f33fa596d9075feb57218" [3mcandidate_count[0m[2m=[0m2
[pod/payload-processing-external-model-8657f8dff8-j9qn4/payload-processing] [2m2026-11-03T17:40:56.584836Z[0m [32m INFO[0m [2mpraxis_ai_filters::routing::overlay[0m[2m:[0m intelligent_route: overlay file watcher started [3mpath[0m[2m=[0m/etc/praxis/routing/routing-overlay.json [3mdebounce_ms[0m[2m=[0m500
[pod/payload-processing-external-model-8657f8dff8-j9qn4/payload-processing] [2m2026-11-03T17:40:56.585867Z[0m [32m INFO[0m [2mpraxis_extproc[0m[2m:[0m starting ExtProc server [3mgrpc[0m[2m=[0m0.0.0.0:9004 [3mhealth[0m[2m=[0m0.0.0.0:50052 [3mmetrics[0m[2m=[0m0.0.0.0:9090 [3mfilters[0m[2m=[0m3
[pod/payload-processing-external-model-8657f8dff8-j9qn4/payload-processing] [2m2026-11-03T17:40:56.585902Z[0m [32m INFO[0m [2mpraxis_extproc[0m[2m:[0m gRPC max decoding message size configured [3mbytes[0m[2m=[0m33554432 [3menv[0m[2m=[0m"GRPC_MAX_RECV_MSG_SIZE"
[pod/payload-processing-external-model-8657f8dff8-j9qn4/payload-processing] [2m2026-11-03T17:40:56.585909Z[0m [32m INFO[0m [2mpraxis_extproc::tls[0m[2m:[0m generating self-signed TLS certificate
[pod/payload-processing-external-model-8657f8dff8-j9qn4/payload-processing] [2m2026-11-03T17:40:56.585925Z[0m [32m INFO[0m [2mpraxis_extproc::health[0m[2m:[0m health server listening [3maddress[0m[2m=[0m0.0.0.0:50052 [3mserving[0m[2m=[0mtrue [3mfips[0m[2m=[0mfalse
[pod/payload-processing-external-model-8657f8dff8-j9qn4/payload-processing] [2m2026-11-03T17:40:56.586766Z[0m [32m INFO[0m [2mpraxis_extproc::metrics[0m[2m:[0m metrics server listening [3maddress[0m[2m=[0m0.0.0.0:9090