tests/test_tenant_external_model_conflicts.py::TestForeignBrokenModelIsolation::test_tenant_models_serve_after_broken_model_deleted@tenant_ipp
{"namespace": "ai-tenant-e2e-extmod-a-6ac03f", "pod": "payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv", "status": {"conditions": [{"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:51:56Z", "observedGeneration": 1, "status": "True", "type": "PodReadyToStartContainers"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:51:55Z", "observedGeneration": 1, "status": "True", "type": "Initialized"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:52:07Z", "observedGeneration": 1, "status": "True", "type": "Ready"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:52:07Z", "observedGeneration": 1, "status": "True", "type": "ContainersReady"}, {"lastProbeTime": null, "lastTransitionTime": "2026-11-02T12:51:55Z", "observedGeneration": 1, "status": "True", "type": "PodScheduled"}], "containerStatuses": [{"allocatedResources": {"cpu": "50m", "memory": "64Mi"}, "containerID": "cri-o://179203cebe26d69237a8393aec8f020f006fa16401523c049920a1f3ef9b1771", "image": "quay.io/opendatahub/odh-praxis-extproc:odh-stable", "imageID": "quay.io/opendatahub/odh-praxis-extproc@sha256:8b7084478cc266b2373bbfb04190ed3ca6fe759428663bddbab63b9c3ad8f130", "lastState": {}, "name": "payload-processing", "ready": true, "resources": {"limits": {"cpu": "500m", "memory": "256Mi"}, "requests": {"cpu": "50m", "memory": "64Mi"}}, "restartCount": 0, "started": true, "state": {"running": {"startedAt": "2026-11-02T12:51:56Z"}}, "user": {"linux": {"gid": 0, "supplementalGroups": [0, 1000840000], "uid": 1000840000}}, "volumeMounts": [{"mountPath": "/etc/praxis", "name": "plugins-config-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/routing", "name": "routing-overlay-volume", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/etc/praxis/credentials", "name": "provider-credentials", "readOnly": true, "recursiveReadOnly": "Disabled"}, {"mountPath": "/var/run/secrets/kubernetes.io/serviceaccount", "name": "kube-api-access-ft9zd", "readOnly": true, "recursiveReadOnly": "Disabled"}]}], "hostIP": "10.0.133.239", "hostIPs": [{"ip": "10.0.133.239"}], "observedGeneration": 1, "phase": "Running", "podIP": "10.133.0.223", "podIPs": [{"ip": "10.133.0.223"}], "qosClass": "Burstable", "startTime": "2026-11-02T12:51:55Z"}}
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:51:56.508730Z  INFO praxis_extproc::fips: installed rustls crypto provider provider="openssl" provider_fips=false kernel_fips=Some(false) fips_required=false
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:51:56.516598Z  INFO praxis_ai_filters::routing::intelligent_route: intelligent_route: overlay snapshot initialized path=/etc/praxis/routing/routing-overlay.json contract_format=Envelope schema_version="1.0.0" accepted_revision="02dc26f6dbeac7779d7658b96ddeb0d14d6005e899e4ce579a3ae0ed1220f706" serving_revision="02dc26f6dbeac7779d7658b96ddeb0d14d6005e899e4ce579a3ae0ed1220f706" candidate_count=5
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:51:56.516890Z  INFO praxis_ai_filters::routing::overlay: intelligent_route: overlay file watcher started path=/etc/praxis/routing/routing-overlay.json debounce_ms=500
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:51:56.518524Z  INFO praxis_extproc: starting ExtProc server grpc=0.0.0.0:9004 health=0.0.0.0:50052 metrics=0.0.0.0:9090 filters=3
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:51:56.518558Z  INFO praxis_extproc: gRPC max decoding message size configured bytes=33554432 env="GRPC_MAX_RECV_MSG_SIZE"
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:51:56.518566Z  INFO praxis_extproc::tls: generating self-signed TLS certificate
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:51:56.518594Z  INFO praxis_extproc::health: health server listening address=0.0.0.0:50052 serving=true fips=false
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:51:56.519445Z  INFO praxis_extproc::metrics: metrics server listening address=0.0.0.0:9090
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:52:16.845003Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:52:22.263708Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:52:27.515562Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:52:32.768388Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:52:38.028308Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:52:43.280803Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:52:48.537005Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:52:53.966032Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:52:59.223398Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:53:04.471511Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:53:09.724044Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:53:14.979384Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:53:20.234251Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:53:25.672415Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:53:30.927561Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:53:36.179655Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:53:41.426859Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:53:46.676015Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:53:51.931509Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:53:57.354355Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:54:02.609268Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:54:07.877166Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:54:13.128730Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:54:18.394792Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:54:23.644067Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:54:29.064951Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:54:34.313914Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:54:39.563094Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:54:44.813632Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:54:50.070015Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:54:55.333501Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:55:00.770308Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:55:06.029591Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:55:11.288665Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:55:16.551432Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:55:21.806722Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:55:23.330775Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:55:28.743517Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:55:33.995927Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:55:39.249205Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:55:44.500098Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:55:49.750926Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:55:55.012008Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:56:00.440606Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:56:05.693202Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:56:10.942790Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:56:16.194747Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:56:21.461961Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:56:26.727701Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:56:32.144708Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:56:37.396088Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:56:42.651398Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:56:47.904778Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:56:53.158273Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:56:58.410906Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:57:03.838254Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:57:09.092778Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:57:14.349312Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:57:19.608062Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:57:24.866657Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:57:30.115419Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:57:35.536196Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:57:40.791971Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:57:46.038618Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:57:51.289792Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:57:56.538368Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:58:01.787785Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:58:07.214569Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:58:12.472840Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:58:17.722830Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:58:22.981718Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503
[pod/payload-processing-external-model-e2e-extmod-a-6ac03f-5846gtdtv/payload-processing] 2026-11-02T12:58:28.233890Z  WARN praxis_filter::pipeline::http_utils: request rejected by filter filter="credential_inject" status=503