tests/test_tenant_external_model_attachment.py::TestCredentialRotationFollowsSource::test_rotation_updates_copy_and_serving@tenant_ipp
{"namespace": "ai-tenant-e2e-extmod-b-3a78b4", "pod": "payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc", "status": {"conditions": [{"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:34:46Z", "observedGeneration": 1, "status": "True", "type": "PodReadyToStartContainers"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:34:45Z", "observedGeneration": 1, "status": "True", "type": "Initialized"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:34:57Z", "observedGeneration": 1, "status": "True", "type": "Ready"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:34:57Z", "observedGeneration": 1, "status": "True", "type": "ContainersReady"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:34:45Z", "observedGeneration": 1, "status": "True", "type": "PodScheduled"}], "containerStatuses": [{"allocatedResources": {"cpu": "50m", "memory": "64Mi"}, "containerID": "cri-o://968b902c8dd5fcd9b1db214102e29faf85a21c92911504f0d5ef183f8d0681f6", "image": "quay.io/opendatahub/odh-praxis-extproc:odh-stable", "imageID": "quay.io/opendatahub/odh-praxis-extproc@sha256:8b7084478cc266b2373bbfb04190ed3ca6fe759428663bddbab63b9c3ad8f130", "lastState": {}, "name": "payload-processing", "ready": true, "resources": {"limits": {"cpu": "500m", "memory": "256Mi"}, "requests": {"cpu": "50m", "memory": "64Mi"}}, "restartCount": 0, "started": true, "state": {"running": {"startedAt": "2026-11-02T12:34:46Z"}}, "user": {"linux": {"gid": 0, "supplementalGroups": [0, 1000850000], "uid": 1000850000}}, "volumeMounts": [{"mountPath": "/etc/praxis", "name": "plugins-config-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/routing", "name": "routing-overlay-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/credentials", "name": "provider-credentials", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", "name": "kube-api-access-xlfpn", "readOnly": true, "recursiveReadOnly": "Disabled"}]}], "hostIP": "10.0.141.170", "hostIPs": [{"ip": "10.0.141.170"}], "observedGeneration": 1, "phase": "Running", "podIP": "10.134.0.94", "podIPs": [{"ip": "10.134.0.94"}], "qosClass": "Burstable", "startTime": "2026-11-02T12:34:45Z"}}
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:34:46.157403Z[0m [32m INFO[0m [2mpraxis_extproc::fips[0m[2m:[0m installed rustls crypto provider [3mprovider[0m[2m=[0m"openssl" [3mprovider_fips[0m[2m=[0mfalse [3mkernel_fips[0m[2m=[0mSome(false) [3mfips_required[0m[2m=[0mfalse
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:34:46.164847Z[0m [32m INFO[0m [2mpraxis_ai_filters::routing::intelligent_route[0m[2m:[0m intelligent_route: overlay snapshot initialized [3mpath[0m[2m=[0m/etc/praxis/routing/routing-overlay.json [3mcontract_format[0m[2m=[0mEnvelope [3mschema_version[0m[2m=[0m"1.0.0" [3maccepted_revision[0m[2m=[0m"9c936e5c5a185f0af58171caeebe2a49d996e3ff6970310cca60039381ec8ff4" [3mserving_revision[0m[2m=[0m"9c936e5c5a185f0af58171caeebe2a49d996e3ff6970310cca60039381ec8ff4" [3mcandidate_count[0m[2m=[0m4
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:34:46.165173Z[0m [32m INFO[0m [2mpraxis_ai_filters::routing::overlay[0m[2m:[0m intelligent_route: overlay file watcher started [3mpath[0m[2m=[0m/etc/praxis/routing/routing-overlay.json [3mdebounce_ms[0m[2m=[0m500
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:34:46.166426Z[0m [32m INFO[0m [2mpraxis_extproc[0m[2m:[0m starting ExtProc server [3mgrpc[0m[2m=[0m0.0.0.0:9004 [3mhealth[0m[2m=[0m0.0.0.0:50052 [3mmetrics[0m[2m=[0m0.0.0.0:9090 [3mfilters[0m[2m=[0m3
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:34:46.166452Z[0m [32m INFO[0m [2mpraxis_extproc[0m[2m:[0m gRPC max decoding message size configured [3mbytes[0m[2m=[0m33554432 [3menv[0m[2m=[0m"GRPC_MAX_RECV_MSG_SIZE"
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:34:46.166457Z[0m [32m INFO[0m [2mpraxis_extproc::tls[0m[2m:[0m generating self-signed TLS certificate
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:34:46.166472Z[0m [32m INFO[0m [2mpraxis_extproc::health[0m[2m:[0m health server listening [3maddress[0m[2m=[0m0.0.0.0:50052 [3mserving[0m[2m=[0mtrue [3mfips[0m[2m=[0mfalse
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:34:46.167201Z[0m [32m INFO[0m [2mpraxis_extproc::metrics[0m[2m:[0m metrics server listening [3maddress[0m[2m=[0m0.0.0.0:9090
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:35:05.836443Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:35:11.155452Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:35:16.422948Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:35:21.700685Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:35:26.993964Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:35:32.296305Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:35:37.727983Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:35:42.982150Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:35:48.235259Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:35:53.504225Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:35:58.754083Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:36:04.029030Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:36:09.452095Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:36:14.707182Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:36:19.966063Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:36:25.223551Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:36:30.483434Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:36:35.735683Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:36:41.151735Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:36:46.412621Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:36:51.670643Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:36:56.928672Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:37:02.193723Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:37:07.449150Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:37:12.855610Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:37:18.117820Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:37:23.397191Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:37:28.660706Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:37:33.916178Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:37:39.171269Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:37:44.581621Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:37:49.839800Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:37:55.090460Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:38:00.341794Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503
[pod/payload-processing-external-model-e2e-extmod-b-3a78b4-55ff8gmhc/payload-processing] [2m2026-11-02T12:38:05.609559Z[0m [33m WARN[0m [2mpraxis_filter::pipeline::http_utils[0m[2m:[0m request rejected by filter [3mfilter[0m[2m=[0m"credential_inject" [3mstatus[0m[2m=[0m503