+ bash
+ STATUS_FILE=/test-status/deploy-and-e2e-status
+ echo failed
+ COMPONENT_NAME=kserve-agent-ci
++ jq -r --arg component_name kserve-agent-ci '.[$component_name].image'
+ export KSERVE_AGENT_IMAGE=quay.io/opendatahub/kserve-agent@sha256:f2fb11757d99ef46b1388e1609ace6b5f516fdebd4ee81c172186c5de9dc18b9
+ KSERVE_AGENT_IMAGE=quay.io/opendatahub/kserve-agent@sha256:f2fb11757d99ef46b1388e1609ace6b5f516fdebd4ee81c172186c5de9dc18b9
+ COMPONENT_NAME=kserve-controller-ci
++ jq -r --arg component_name kserve-controller-ci '.[$component_name].image'
+ export KSERVE_CONTROLLER_IMAGE=quay.io/opendatahub/kserve-controller@sha256:221297a037331b254cb64387bd19f2c6712adddcbdf267cc0cf14c83f2b79496
+ KSERVE_CONTROLLER_IMAGE=quay.io/opendatahub/kserve-controller@sha256:221297a037331b254cb64387bd19f2c6712adddcbdf267cc0cf14c83f2b79496
+ COMPONENT_NAME=kserve-router-ci
++ jq -r --arg component_name kserve-router-ci '.[$component_name].image'
+ export KSERVE_ROUTER_IMAGE=quay.io/opendatahub/kserve-router@sha256:743cff1ceb833e46d1b8e65ba261d1d549e2e90054da6d533b54a9a2f13462c7
+ KSERVE_ROUTER_IMAGE=quay.io/opendatahub/kserve-router@sha256:743cff1ceb833e46d1b8e65ba261d1d549e2e90054da6d533b54a9a2f13462c7
+ COMPONENT_NAME=kserve-storage-initializer-ci
++ jq -r --arg component_name kserve-storage-initializer-ci '.[$component_name].image'
+ export STORAGE_INITIALIZER_IMAGE=quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840
+ STORAGE_INITIALIZER_IMAGE=quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840
+ export GITHUB_SHA=stable
+ GITHUB_SHA=stable
+ ./test/scripts/openshift-ci/run-e2e-tests.sh graph 2 raw
Installing on cluster
Using namespace: kserve for KServe components
SKLEARN_IMAGE=quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1326
KSERVE_CONTROLLER_IMAGE=quay.io/opendatahub/kserve-controller@sha256:221297a037331b254cb64387bd19f2c6712adddcbdf267cc0cf14c83f2b79496
LLMISVC_CONTROLLER_IMAGE=ghcr.io/opendatahub-io/kserve/odh-kserve-llmisvc-controller:release-v0.17
KSERVE_AGENT_IMAGE=quay.io/opendatahub/kserve-agent@sha256:f2fb11757d99ef46b1388e1609ace6b5f516fdebd4ee81c172186c5de9dc18b9
KSERVE_ROUTER_IMAGE=quay.io/opendatahub/kserve-router@sha256:743cff1ceb833e46d1b8e65ba261d1d549e2e90054da6d533b54a9a2f13462c7
STORAGE_INITIALIZER_IMAGE=quay.io/opendatahub/kserve-storage-initializer@sha256:0784282e0aaaff940f9515f44b229c4f93907c9657748d25ae94f59437bac840
ERROR_404_ISVC_IMAGE=quay.io/opendatahub/error-404-isvc:odh-pr-1326
SUCCESS_200_ISVC_IMAGE=quay.io/opendatahub/success-200-isvc:odh-pr-1326
[INFO] Installing Kustomize v5.8.1 for linux/amd64...
[SUCCESS] Successfully installed Kustomize v5.8.1 to /workspace/source/bin/kustomize
v5.8.1
make: Entering directory '/workspace/source'
[INFO] Installing yq v4.52.1 for linux/amd64...
[SUCCESS] Successfully installed yq v4.52.1 to /workspace/source/bin/yq
yq (https://github.com/mikefarah/yq/) version v4.52.1
make: Leaving directory '/workspace/source'
Installing KServe Python SDK ...
[INFO] Installing uv 0.7.8 for linux/amd64...
[SUCCESS] Successfully installed uv 0.7.8 to /workspace/source/bin/uv
warning: Failed to read project metadata (No `pyproject.toml` found in current directory or any parent directory). Running `uv self version` for compatibility. This fallback will be removed in the future; pass `--preview` to force an error.
uv 0.7.8
Creating virtual environment...
warning: virtualenv's `--clear` has no effect (uv always clears the virtual environment)
Using CPython 3.9.25 interpreter at: /usr/bin/python3
Creating virtual environment at: .venv
/workspace/source
Using CPython 3.11.13 interpreter at: /usr/bin/python3.11
Creating virtual environment at: .venv
Resolved 263 packages in 1ms
Building kserve @ file:///workspace/source/python/kserve
Downloading aiohttp (1.7MiB)
Downloading kubernetes (1.9MiB)
Downloading pydantic-core (2.0MiB)
Downloading uvloop (3.8MiB)
Downloading cryptography (4.3MiB)
Downloading mypy (17.2MiB)
Downloading pandas (12.5MiB)
Downloading portforward (3.9MiB)
Downloading pyarrow (40.1MiB)
Downloading black (1.6MiB)
Downloading botocore (12.9MiB)
Downloading grpcio-tools (2.5MiB)
Downloading grpcio (6.4MiB)
Downloading numpy (15.7MiB)
Downloading setuptools (1.2MiB)
Building timeout-sampler==1.0.3
Building python-simple-logger==2.0.19
Downloading aiohttp
Downloading pydantic-core
Downloading black
Downloading grpcio-tools
Downloading setuptools
Built python-simple-logger==2.0.19
Downloading portforward
Downloading uvloop
Downloading cryptography
Downloading grpcio
Downloading kubernetes
Built timeout-sampler==1.0.3
Built kserve @ file:///workspace/source/python/kserve
Downloading numpy
Downloading pandas
Downloading botocore
Downloading mypy
Downloading pyarrow
Prepared 99 packages in 1.86s
warning: Failed to hardlink files; falling back to full copy. This may lead to degraded performance.
If the cache and target directories are on different filesystems, hardlinking may not be supported.
If this is intentional, set `export UV_LINK_MODE=copy` or use `--link-mode=copy` to suppress this warning.
Installed 99 packages in 262ms
+ aiohappyeyeballs==2.6.1
+ aiohttp==3.13.3
+ aiosignal==1.4.0
+ annotated-doc==0.0.4
+ annotated-types==0.7.0
+ anyio==4.9.0
+ attrs==25.3.0
+ avro==1.12.0
+ black==24.3.0
+ boto3==1.37.35
+ botocore==1.37.35
+ cachetools==5.5.2
+ certifi==2025.1.31
+ cffi==2.0.0
+ charset-normalizer==3.4.1
+ click==8.1.8
+ cloudevents==1.11.0
+ colorama==0.4.6
+ colorlog==6.10.1
+ coverage==7.8.0
+ cryptography==46.0.5
+ deprecation==2.1.0
+ durationpy==0.9
+ execnet==2.1.1
+ fastapi==0.121.3
+ frozenlist==1.5.0
+ google-auth==2.39.0
+ grpc-interceptor==0.15.4
+ grpcio==1.78.1
+ grpcio-testing==1.78.1
+ grpcio-tools==1.78.1
+ h11==0.16.0
+ httpcore==1.0.9
+ httptools==0.6.4
+ httpx==0.27.2
+ httpx-retries==0.4.5
+ idna==3.10
+ iniconfig==2.1.0
+ jinja2==3.1.6
+ jmespath==1.0.1
+ kserve==0.17.0 (from file:///workspace/source/python/kserve)
+ kubernetes==32.0.1
+ markupsafe==3.0.2
+ multidict==6.4.3
+ mypy==0.991
+ mypy-extensions==1.0.0
+ numpy==2.2.4
+ oauthlib==3.2.2
+ orjson==3.10.16
+ packaging==24.2
+ pandas==2.2.3
+ pathspec==0.12.1
+ platformdirs==4.3.7
+ pluggy==1.5.0
+ portforward==0.7.1
+ prometheus-client==0.21.1
+ propcache==0.3.1
+ protobuf==6.33.5
+ psutil==5.9.8
+ pyarrow==19.0.1
+ pyasn1==0.6.1
+ pyasn1-modules==0.4.2
+ pycparser==2.22
+ pydantic==2.12.4
+ pydantic-core==2.41.5
+ pyjwt==2.12.1
+ pytest==7.4.4
+ pytest-asyncio==0.23.8
+ pytest-cov==5.0.0
+ pytest-httpx==0.30.0
+ pytest-xdist==3.6.1
+ python-dateutil==2.9.0.post0
+ python-dotenv==1.1.0
+ python-multipart==0.0.22
+ python-simple-logger==2.0.19
+ pytz==2025.2
+ pyyaml==6.0.2
+ requests==2.32.3
+ requests-oauthlib==2.0.0
+ rsa==4.9.1
+ s3transfer==0.11.4
+ setuptools==78.1.0
+ six==1.17.0
+ sniffio==1.3.1
+ starlette==0.49.1
+ tabulate==0.9.0
+ timeout-sampler==1.0.3
+ timing-asgi==0.3.1
+ tomlkit==0.13.2
+ typing-extensions==4.15.0
+ typing-inspection==0.4.2
+ tzdata==2025.2
+ urllib3==2.6.2
+ uvicorn==0.34.1
+ uvloop==0.21.0
+ watchfiles==1.0.5
+ websocket-client==1.8.0
+ websockets==15.0.1
+ yarl==1.20.0
Audited 1 package in 47ms
/workspace/source
Creating namespace openshift-keda...
namespace/openshift-keda created
Namespace openshift-keda created/ensured.
---
Creating OperatorGroup openshift-keda...
operatorgroup.operators.coreos.com/openshift-keda created
OperatorGroup openshift-keda created/ensured.
---
Creating Subscription for openshift-custom-metrics-autoscaler-operator...
subscription.operators.coreos.com/openshift-custom-metrics-autoscaler-operator created
Subscription openshift-custom-metrics-autoscaler-operator created/ensured.
---
Waiting for openshift-custom-metrics-autoscaler-operator CSV to become ready...
Waiting for CSV to be installed for subscription openshift-custom-metrics-autoscaler-operator... (0/600)
Waiting for CSV to be installed for subscription openshift-custom-metrics-autoscaler-operator... (5/600)
Waiting for CSV to be installed for subscription openshift-custom-metrics-autoscaler-operator... (10/600)
Waiting for CSV to be installed for subscription openshift-custom-metrics-autoscaler-operator... (15/600)
Waiting for CSV to be installed for subscription openshift-custom-metrics-autoscaler-operator... (20/600)
CSV custom-metrics-autoscaler.v2.18.1-2 found, but not yet Succeeded (Phase: Installing). Waiting... (25/600)
CSV custom-metrics-autoscaler.v2.18.1-2 found, but not yet Succeeded (Phase: Installing). Waiting... (30/600)
CSV custom-metrics-autoscaler.v2.18.1-2 found, but not yet Succeeded (Phase: Installing). Waiting... (35/600)
CSV custom-metrics-autoscaler.v2.18.1-2 found, but not yet Succeeded (Phase: Installing). Waiting... (40/600)
CSV custom-metrics-autoscaler.v2.18.1-2 found, but not yet Succeeded (Phase: Installing). Waiting... (45/600)
CSV custom-metrics-autoscaler.v2.18.1-2 is ready (Phase: Succeeded).
---
Applying KedaController custom resource...
Warning: resource kedacontrollers/keda is missing the kubectl.kubernetes.io/last-applied-configuration annotation which is required by oc apply. oc apply should only be used on resources created declaratively by either oc create --save-config or oc apply. The missing annotation will be patched automatically.
kedacontroller.keda.sh/keda configured
KedaController custom resource applied.
---
Allowing time for KEDA components to be provisioned by the operator ...
Waiting for KEDA Operator pod (selector: "app=keda-operator") to be ready in namespace openshift-keda...
Waiting for pod -l "app=keda-operator" in namespace "openshift-keda" to be created...
Pod -l "app=keda-operator" in namespace "openshift-keda" found.
Current pods for -l "app=keda-operator" in namespace "openshift-keda":
NAME READY STATUS RESTARTS AGE
keda-operator-ffbb595cb-8wb5z 1/1 Running 0 43s
Waiting up to 120s for pod(s) -l "app=keda-operator" in namespace "openshift-keda" to become ready...
pod/keda-operator-ffbb595cb-8wb5z condition met
Pod(s) -l "app=keda-operator" in namespace "openshift-keda" are ready.
KEDA Operator pod is ready.
Waiting for KEDA Metrics API Server pod (selector: "app=keda-metrics-apiserver") to be ready in namespace openshift-keda...
Waiting for pod -l "app=keda-metrics-apiserver" in namespace "openshift-keda" to be created...
Pod -l "app=keda-metrics-apiserver" in namespace "openshift-keda" found.
Current pods for -l "app=keda-metrics-apiserver" in namespace "openshift-keda":
NAME READY STATUS RESTARTS AGE
keda-metrics-apiserver-7c9f485588-j8gww 1/1 Running 0 47s
Waiting up to 120s for pod(s) -l "app=keda-metrics-apiserver" in namespace "openshift-keda" to become ready...
pod/keda-metrics-apiserver-7c9f485588-j8gww condition met
Pod(s) -l "app=keda-metrics-apiserver" in namespace "openshift-keda" are ready.
KEDA Metrics API Server pod is ready.
Waiting for KEDA Webhook pod (selector: "app=keda-admission-webhooks") to be ready in namespace openshift-keda...
Waiting for pod -l "app=keda-admission-webhooks" in namespace "openshift-keda" to be created...
Pod -l "app=keda-admission-webhooks" in namespace "openshift-keda" found.
Current pods for -l "app=keda-admission-webhooks" in namespace "openshift-keda":
NAME READY STATUS RESTARTS AGE
keda-admission-cf49989db-gdq8x 1/1 Running 0 52s
Waiting up to 120s for pod(s) -l "app=keda-admission-webhooks" in namespace "openshift-keda" to become ready...
pod/keda-admission-cf49989db-gdq8x condition met
Pod(s) -l "app=keda-admission-webhooks" in namespace "openshift-keda" are ready.
KEDA Webhook pod is ready.
---
✅ KEDA deployment script finished successfully.
Now using project "kserve" on server "https://a9b56e18e9e4c45dcaeda4bb15f69bb8-9f16fbcdf636ba83.elb.us-east-1.amazonaws.com:6443".
You can add applications to this project with the 'new-app' command. For example, try:
oc new-app rails-postgresql-example
to build a new example application in Ruby. Or use kubectl to deploy a simple Kubernetes application:
kubectl create deployment hello-node --image=registry.k8s.io/e2e-test-images/agnhost:2.43 -- /agnhost serve-hostname
⏳ Installing KServe with SeaweedFS
# Warning: 'commonLabels' is deprecated. Please use 'labels' instead. Run 'kustomize edit fix' to update your Kustomization automatically.
customresourcedefinition.apiextensions.k8s.io/clusterstoragecontainers.serving.kserve.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/datascienceclusters.datasciencecluster.opendatahub.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/dscinitializations.dscinitialization.opendatahub.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/inferencegraphs.serving.kserve.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/inferencemodelrewrites.inference.networking.x-k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/inferenceobjectives.inference.networking.x-k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/inferencepoolimports.inference.networking.x-k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/inferencepools.inference.networking.k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/inferencepools.inference.networking.x-k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/inferenceservices.serving.kserve.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/llminferenceserviceconfigs.serving.kserve.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/llminferenceservices.serving.kserve.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/servingruntimes.serving.kserve.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/trainedmodels.serving.kserve.io serverside-applied
⏳ Waiting for CRDs to be established
Waiting for CRD inferenceservices.serving.kserve.io to appear (timeout: 90s)…
CRD inferenceservices.serving.kserve.io detected — waiting for it to become Established (timeout: 90s)…
customresourcedefinition.apiextensions.k8s.io/inferenceservices.serving.kserve.io condition met
Waiting for CRD llminferenceserviceconfigs.serving.kserve.io to appear (timeout: 90s)…
CRD llminferenceserviceconfigs.serving.kserve.io detected — waiting for it to become Established (timeout: 90s)…
customresourcedefinition.apiextensions.k8s.io/llminferenceserviceconfigs.serving.kserve.io condition met
Waiting for CRD clusterstoragecontainers.serving.kserve.io to appear (timeout: 90s)…
CRD clusterstoragecontainers.serving.kserve.io detected — waiting for it to become Established (timeout: 90s)…
customresourcedefinition.apiextensions.k8s.io/clusterstoragecontainers.serving.kserve.io condition met
Waiting for CRD datascienceclusters.datasciencecluster.opendatahub.io to appear (timeout: 90s)…
CRD datascienceclusters.datasciencecluster.opendatahub.io detected — waiting for it to become Established (timeout: 90s)…
customresourcedefinition.apiextensions.k8s.io/datascienceclusters.datasciencecluster.opendatahub.io condition met
⏳ Applying all resources...
customresourcedefinition.apiextensions.k8s.io/clusterstoragecontainers.serving.kserve.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/datascienceclusters.datasciencecluster.opendatahub.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/dscinitializations.dscinitialization.opendatahub.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/inferencegraphs.serving.kserve.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/inferencemodelrewrites.inference.networking.x-k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/inferenceobjectives.inference.networking.x-k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/inferencepoolimports.inference.networking.x-k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/inferencepools.inference.networking.k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/inferencepools.inference.networking.x-k8s.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/inferenceservices.serving.kserve.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/llminferenceserviceconfigs.serving.kserve.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/llminferenceservices.serving.kserve.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/servingruntimes.serving.kserve.io serverside-applied
customresourcedefinition.apiextensions.k8s.io/trainedmodels.serving.kserve.io serverside-applied
serviceaccount/kserve-controller-manager serverside-applied
serviceaccount/llmisvc-controller-manager serverside-applied
role.rbac.authorization.k8s.io/kserve-leader-election-role serverside-applied
role.rbac.authorization.k8s.io/llmisvc-leader-election-role serverside-applied
clusterrole.rbac.authorization.k8s.io/kserve-admin serverside-applied
clusterrole.rbac.authorization.k8s.io/kserve-edit serverside-applied
clusterrole.rbac.authorization.k8s.io/kserve-llmisvc-distro-role serverside-applied
clusterrole.rbac.authorization.k8s.io/kserve-llmisvc-manager-role serverside-applied
clusterrole.rbac.authorization.k8s.io/kserve-manager-role serverside-applied
clusterrole.rbac.authorization.k8s.io/kserve-metrics-reader-cluster-role serverside-applied
clusterrole.rbac.authorization.k8s.io/kserve-proxy-role serverside-applied
clusterrole.rbac.authorization.k8s.io/kserve-view serverside-applied
clusterrole.rbac.authorization.k8s.io/openshift-ai-llminferenceservice-scc serverside-applied
rolebinding.rbac.authorization.k8s.io/kserve-leader-election-rolebinding serverside-applied
rolebinding.rbac.authorization.k8s.io/llmisvc-leader-election-rolebinding serverside-applied
clusterrolebinding.rbac.authorization.k8s.io/kserve-llmisvc-distro-rolebinding serverside-applied
clusterrolebinding.rbac.authorization.k8s.io/kserve-manager-rolebinding serverside-applied
clusterrolebinding.rbac.authorization.k8s.io/kserve-proxy-rolebinding serverside-applied
clusterrolebinding.rbac.authorization.k8s.io/llmisvc-manager-rolebinding serverside-applied
configmap/inferenceservice-config serverside-applied
configmap/kserve-parameters serverside-applied
secret/kserve-webhook-server-secret serverside-applied
secret/mlpipeline-s3-artifact serverside-applied
service/kserve-controller-manager-metrics-service serverside-applied
service/kserve-controller-manager-service serverside-applied
service/kserve-webhook-server-service serverside-applied
service/llmisvc-controller-manager-service serverside-applied
service/llmisvc-webhook-server-service serverside-applied
service/s3-service serverside-applied
deployment.apps/kserve-controller-manager serverside-applied
deployment.apps/llmisvc-controller-manager serverside-applied
deployment.apps/seaweedfs serverside-applied
networkpolicy.networking.k8s.io/kserve-controller-manager serverside-applied
securitycontextconstraints.security.openshift.io/openshift-ai-llminferenceservice-scc serverside-applied
clusterstoragecontainer.serving.kserve.io/default serverside-applied
llminferenceserviceconfig.serving.kserve.io/kserve-config-llm-decode-template serverside-applied
llminferenceserviceconfig.serving.kserve.io/kserve-config-llm-decode-worker-data-parallel serverside-applied
llminferenceserviceconfig.serving.kserve.io/kserve-config-llm-prefill-template serverside-applied
llminferenceserviceconfig.serving.kserve.io/kserve-config-llm-prefill-worker-data-parallel serverside-applied
llminferenceserviceconfig.serving.kserve.io/kserve-config-llm-router-route serverside-applied
llminferenceserviceconfig.serving.kserve.io/kserve-config-llm-scheduler serverside-applied
llminferenceserviceconfig.serving.kserve.io/kserve-config-llm-template serverside-applied
llminferenceserviceconfig.serving.kserve.io/kserve-config-llm-template-amd-rocm serverside-applied
llminferenceserviceconfig.serving.kserve.io/kserve-config-llm-template-ibm-spyre-ppc64le serverside-applied
llminferenceserviceconfig.serving.kserve.io/kserve-config-llm-template-ibm-spyre-s390x serverside-applied
llminferenceserviceconfig.serving.kserve.io/kserve-config-llm-template-ibm-spyre-x86 serverside-applied
llminferenceserviceconfig.serving.kserve.io/kserve-config-llm-template-intel-gaudi serverside-applied
llminferenceserviceconfig.serving.kserve.io/kserve-config-llm-template-nvidia-cuda serverside-applied
llminferenceserviceconfig.serving.kserve.io/kserve-config-llm-worker-data-parallel serverside-applied
mutatingwebhookconfiguration.admissionregistration.k8s.io/inferenceservice.serving.kserve.io serverside-applied
validatingwebhookconfiguration.admissionregistration.k8s.io/inferencegraph.serving.kserve.io serverside-applied
validatingwebhookconfiguration.admissionregistration.k8s.io/inferenceservice.serving.kserve.io serverside-applied
validatingwebhookconfiguration.admissionregistration.k8s.io/llminferenceservice.serving.kserve.io serverside-applied
validatingwebhookconfiguration.admissionregistration.k8s.io/llminferenceserviceconfig.serving.kserve.io serverside-applied
validatingwebhookconfiguration.admissionregistration.k8s.io/servingruntime.serving.kserve.io serverside-applied
validatingwebhookconfiguration.admissionregistration.k8s.io/trainedmodel.serving.kserve.io serverside-applied
⏳ Waiting for llmisvc-controller-manager to be ready...
Waiting for pod -l "control-plane=llmisvc-controller-manager" in namespace "kserve" to be created...
Pod -l "control-plane=llmisvc-controller-manager" in namespace "kserve" found.
Current pods for -l "control-plane=llmisvc-controller-manager" in namespace "kserve":
NAME READY STATUS RESTARTS AGE
llmisvc-controller-manager-68cc5db7c4-d6nmf 0/1 Running 0 6s
Waiting up to 600s for pod(s) -l "control-plane=llmisvc-controller-manager" in namespace "kserve" to become ready...
pod/llmisvc-controller-manager-68cc5db7c4-d6nmf condition met
Pod(s) -l "control-plane=llmisvc-controller-manager" in namespace "kserve" are ready.
⏳ Re-applying LLMInferenceServiceConfig resources with webhook validation...
Warning: modifying well-known config kserve/kserve-config-llm-decode-template is not recommended. Consider creating a custom config instead
llminferenceserviceconfig.serving.kserve.io/kserve-config-llm-decode-template serverside-applied
Warning: modifying well-known config kserve/kserve-config-llm-decode-worker-data-parallel is not recommended. Consider creating a custom config instead
llminferenceserviceconfig.serving.kserve.io/kserve-config-llm-decode-worker-data-parallel serverside-applied
Warning: modifying well-known config kserve/kserve-config-llm-prefill-template is not recommended. Consider creating a custom config instead
llminferenceserviceconfig.serving.kserve.io/kserve-config-llm-prefill-template serverside-applied
Warning: modifying well-known config kserve/kserve-config-llm-prefill-worker-data-parallel is not recommended. Consider creating a custom config instead
llminferenceserviceconfig.serving.kserve.io/kserve-config-llm-prefill-worker-data-parallel serverside-applied
llminferenceserviceconfig.serving.kserve.io/kserve-config-llm-router-route serverside-applied
Warning: modifying well-known config kserve/kserve-config-llm-scheduler is not recommended. Consider creating a custom config instead
llminferenceserviceconfig.serving.kserve.io/kserve-config-llm-scheduler serverside-applied
Warning: modifying well-known config kserve/kserve-config-llm-template is not recommended. Consider creating a custom config instead
llminferenceserviceconfig.serving.kserve.io/kserve-config-llm-template serverside-applied
Warning: modifying well-known config kserve/kserve-config-llm-worker-data-parallel is not recommended. Consider creating a custom config instead
llminferenceserviceconfig.serving.kserve.io/kserve-config-llm-worker-data-parallel serverside-applied
Installing DSC/DSCI resources...
dscinitialization.dscinitialization.opendatahub.io/test-dsci created
datasciencecluster.datasciencecluster.opendatahub.io/test-dsc created
Patching ingress domain, markers: graph
configmap/inferenceservice-config patched
pod "kserve-controller-manager-7c68cb4fc8-8bfrh" deleted
datasciencecluster.datasciencecluster.opendatahub.io/test-dsc patched
waiting kserve-controller get ready...
pod/kserve-controller-manager-7c68cb4fc8-f2s2w condition met
Installing ODH Model Controller manually with PR images
customresourcedefinition.apiextensions.k8s.io/accounts.nim.opendatahub.io created
serviceaccount/model-serving-api created
serviceaccount/odh-model-controller created
role.rbac.authorization.k8s.io/leader-election-role created
clusterrole.rbac.authorization.k8s.io/kserve-prometheus-k8s created
clusterrole.rbac.authorization.k8s.io/metrics-reader created
clusterrole.rbac.authorization.k8s.io/model-serving-api created
clusterrole.rbac.authorization.k8s.io/odh-model-controller-role created
clusterrole.rbac.authorization.k8s.io/proxy-role created
rolebinding.rbac.authorization.k8s.io/leader-election-rolebinding created
clusterrolebinding.rbac.authorization.k8s.io/model-serving-api created
clusterrolebinding.rbac.authorization.k8s.io/odh-model-controller-rolebinding-opendatahub created
clusterrolebinding.rbac.authorization.k8s.io/proxy-rolebinding created
configmap/odh-model-controller-parameters created
service/model-serving-api created
service/odh-model-controller-metrics-service created
service/odh-model-controller-webhook-service created
deployment.apps/model-serving-api created
deployment.apps/odh-model-controller created
servicemonitor.monitoring.coreos.com/model-serving-api-metrics created
servicemonitor.monitoring.coreos.com/odh-model-controller-metrics-monitor created
template.template.openshift.io/guardrails-detector-huggingface-serving-template created
template.template.openshift.io/kserve-ovms created
template.template.openshift.io/mlserver-runtime-template created
template.template.openshift.io/vllm-cpu-runtime-template created
template.template.openshift.io/vllm-cpu-x86-runtime-template created
template.template.openshift.io/vllm-cuda-runtime-template created
template.template.openshift.io/vllm-gaudi-runtime-template created
template.template.openshift.io/vllm-multinode-runtime-template created
template.template.openshift.io/vllm-rocm-runtime-template created
template.template.openshift.io/vllm-spyre-ppc64le-runtime-template created
template.template.openshift.io/vllm-spyre-s390x-runtime-template created
template.template.openshift.io/vllm-spyre-x86-runtime-template created
mutatingwebhookconfiguration.admissionregistration.k8s.io/mutating.odh-model-controller.opendatahub.io created
validatingwebhookconfiguration.admissionregistration.k8s.io/validating.odh-model-controller.opendatahub.io created
Waiting for deployment "odh-model-controller" rollout to finish: 0 of 1 updated replicas are available...
deployment "odh-model-controller" successfully rolled out
Add testing models to SeaweedFS S3 storage ...
Waiting for SeaweedFS deployment to be ready...
deployment "seaweedfs" successfully rolled out
S3 init job not completed, re-creating...
job.batch/s3-init created
Waiting for S3 init job to complete...
job.batch/s3-init condition met
networkpolicy.networking.k8s.io/allow-all created
Prepare CI namespace and install ServingRuntimes
Setting up CI namespace: kserve-ci-e2e-test
Tearing down CI namespace: kserve-ci-e2e-test
Namespace kserve-ci-e2e-test does not exist, skipping deletion
CI namespace teardown complete
Creating namespace kserve-ci-e2e-test
namespace/kserve-ci-e2e-test created
Applying S3 artifact secret
secret/mlpipeline-s3-artifact created
Applying storage-config secret
secret/storage-config created
Creating odh-trusted-ca-bundle configmap
configmap/odh-trusted-ca-bundle created
Installing ServingRuntimes
servingruntime.serving.kserve.io/kserve-huggingfaceserver created
servingruntime.serving.kserve.io/kserve-huggingfaceserver-multinode created
servingruntime.serving.kserve.io/kserve-lgbserver created
servingruntime.serving.kserve.io/kserve-mlserver created
servingruntime.serving.kserve.io/kserve-paddleserver created
servingruntime.serving.kserve.io/kserve-pmmlserver created
servingruntime.serving.kserve.io/kserve-predictiveserver created
servingruntime.serving.kserve.io/kserve-sklearnserver created
servingruntime.serving.kserve.io/kserve-tensorflow-serving created
servingruntime.serving.kserve.io/kserve-torchserve created
servingruntime.serving.kserve.io/kserve-tritonserver created
servingruntime.serving.kserve.io/kserve-xgbserver created
CI namespace setup complete
Setup complete
=== E2E cluster / operator summary ===
Client Version: 4.20.11
Kustomize Version: v5.6.0
Server Version: 4.20.18
Kubernetes Version: v1.33.9
ClusterVersion desired: 4.20.18
ClusterVersion history (latest): 4.20.18 (Completed)
CSVs in openshift-keda:
custom-metrics-autoscaler.v2.18.1-2 Succeeded
CSVs in openshift-operators (ODH / shared operators, filtered):
=== End E2E cluster / operator summary ===
/workspace/source
REQUESTS_CA_BUNDLE=-----BEGIN CERTIFICATE-----
MIIDPDCCAiSgAwIBAgIIJMEvntawfLQwDQYJKoZIhvcNAQELBQAwJjESMBAGA1UE
CxMJb3BlbnNoaWZ0MRAwDgYDVQQDEwdyb290LWNhMB4XDTI2MDQxNjE4MjQxNloX
DTM2MDQxMzE4MjQxNlowJjESMBAGA1UECxMJb3BlbnNoaWZ0MRAwDgYDVQQDEwdy
b290LWNhMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA0akh/w0b5hqe
ZKbhJOZCpIt8tRbXlymPV6NHAiFElnDoXfbRCl6uqJDCkBquURjFi2+BO/3jKO1I
scXwrXRPpfl3VQwlVsxEFH+qAoG2G+oGy35tio/JAERiVDazXIKAwSZfxYjC6yVB
zK3WShL8DwokxcmS4rX1sBwQF+6TUDRteIct8WZoquSN4NC4dmIfYJFNmBdP1BkO
5OMBdI0/4QC12NtH+Pob0qImEe+2ZqA/hJgpJy+LGN08oXuRYb0i6aqPKFeljY+H
GNQPC2ENRWW3nhkE15S6+0Xcs0AfqMHXgHSKWNCHPCG9PsmgHqPG5oHfcc1ifJyE
4+pMlgOsEwIDAQABo24wbDAOBgNVHQ8BAf8EBAMCAqQwDwYDVR0TAQH/BAUwAwEB
/zBJBgNVHQ4EQgRA6AaREnBH+EKCmcVWDRzdYgUu/kNqiz9MtJDMS6ZM7BRjJAgG
A2bRhRHfwL2gjj1+cXX6C0DtfP6abS8bvH7bhTANBgkqhkiG9w0BAQsFAAOCAQEA
jEVc8tHt+ynRh8l0q0vvh0JgCJaemKNSTjEB5rmwTX/s+7LfAxkoI79+EovLjwX5
TnTuUAajYXLpQiXANoDxe8cnd/pTOggxDpQAiyAeKJS420UGKIzp8TJbIJs1nbBC
8VdOd5KDIc9QUldJuc9dfRbtBKengAdD2xbquqVfnD0FrxX6NA9ow6pL5WhHK0Bk
89cSNbaMH6vjK11FUQsCE4XzmZCyxqz9X5JuGGEz20uzUeHwgdKWCzdR7kmSj8pc
dhHCh4milTwznUgPT4YKLCM+np9n1UHt55naNWp2HzEobcNsnBPCQzHYL6/7Szik
0cFrYZVbqVEDWctRCLy0hQ==
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIEADCCAuigAwIBAgIITZE7twdu5xQwDQYJKoZIhvcNAQELBQAwJjESMBAGA1UE
CxMJb3BlbnNoaWZ0MRAwDgYDVQQDEwdyb290LWNhMB4XDTI2MDQxNjE4MjQ0M1oX
DTI3MDQxNjE4MjQ0M1owMDESMBAGA1UEChMJb3BlbnNoaWZ0MRowGAYDVQQDExFv
cGVuc2hpZnQtaW5ncmVzczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB
ANFxWyk4vY4sNU8LIIHG58jQf8Td1YN/cKjpsniRbkL27bug6uGaUKrDCFRnQNnG
Ob1J5k7vaP4wh1kG4QmXwXIm5ZPv8pA65mIUGZH66uomml49UGghh+ZBNTQBSwaQ
RjP/IIIRh5FJM8aa68wshIpnuX66l+QHhdeZlaCBLZ0bwDtCYWvvNefcmKLyU9wS
ZgiSKWFzCRZybY42uTJ33Pea1/KoxyNKzhLLmMVtCL+kvOA8/k6Bw/2Adv0zUwiw
UdVnaQ3+YKx2BsnbEueOS4dENHzB1ut+cRD90Kij9iGVIqbrbZUFC80xfbXF4SGf
edEE1hM9FDXTQG5ldbwq2vUCAwEAAaOCASYwggEiMA4GA1UdDwEB/wQEAwIFoDAd
BgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwDAYDVR0TAQH/BAIwADBJBgNV
HQ4EQgRAZZHrPhPDLnoFITHha6hBqekp6j6Cej5mTzSvwzBTxwvjUPJBsXd7ScCy
Wd+knn7cWUkscKvCQloWxegN5piifzBLBgNVHSMERDBCgEDoBpEScEf4QoKZxVYN
HN1iBS7+Q2qLP0y0kMxLpkzsFGMkCAYDZtGFEd/AvaCOPX5xdfoLQO18/pptLxu8
ftuFMEsGA1UdEQREMEKCQCouYXBwcy5iZmUyZGNhNS1iYmZlLTQ3ZTktYmQzZi1h
YzkyMmY1OGJmZWEucHJvZC5rb25mbHV4ZWFhcy5jb20wDQYJKoZIhvcNAQELBQAD
ggEBAGbCLvvSVlHOfIc2ElUXrUmIrsFNqsda8s3pkniUaEI2GuRDIf2/SuACVUIG
ACAvPBV95bnirPXuVQ7AcXZTGyPMN41+JX6Ni6WcMnq+n2YXQoJ7mCOGJ7dBTpIa
2rqHiA+yO5yoRFVy29/Djtl4nS6LS1oIGx8cSBAwsGtyI8klYYZRuvxiIJ6tXcKi
1HH4SNyKLuInlBQaYOCRXSkzI8FpdwV3SAFnbsSABcslUlx9Sc95/qPgLsHfMXAZ
L3RccxgoWkExx2iOAaTmGT8Fl1hiP1qTVYUL0gMk6u+lK/a1B3P8g9N+fkFYuqII
FxJYAjQdLcfCn/CVJdcDfzyB0Uk=
-----END CERTIFICATE-----
-----BEGIN CERTIFICATE-----
MIIDUTCCAjmgAwIBAgIIOYjMwqBzBFQwDQYJKoZIhvcNAQELBQAwNjE0MDIGA1UE
Awwrb3BlbnNoaWZ0LXNlcnZpY2Utc2VydmluZy1zaWduZXJAMTc3NjM2NDMzNzAe
Fw0yNjA0MTYxODMyMTZaFw0yODA2MTQxODMyMTdaMDYxNDAyBgNVBAMMK29wZW5z
aGlmdC1zZXJ2aWNlLXNlcnZpbmctc2lnbmVyQDE3NzYzNjQzMzcwggEiMA0GCSqG
SIb3DQEBAQUAA4IBDwAwggEKAoIBAQC7WZ6dQdTgKfH/J5ecSPtLcGfBNef1iJK0
rI0HrLlAfnd/Lr+JUOskrndU8p+ktianJnOwz0fXe84jag11tka7r3HD7RtFIVGU
ruzNxjxSR8Fzrl6GI8i7gbJUXLRSEzTG3iFKo6Pasn1YVQG7IF538YNVOKhsFQnX
0Y6NLJm0gpuWNlg7eNvgz3MxeDD1gz/mKDT7jor5zOYH0q8LCjwT3ziJO18IB6SH
1nHV1ky7+2UppFWSKWEqi+JYgruc05WyHsIU0OKNGiH4YW+lrag8oOt4e1gwmpPi
XP2WwpAMtZVKVN+B2f6WDW0pTo55XbSjuj8N+Wc5BdMEbHp3OSUrAgMBAAGjYzBh
MA4GA1UdDwEB/wQEAwICpDAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTwZ/PW
KPDjNU7COd/MKJDQxkLpHjAfBgNVHSMEGDAWgBTwZ/PWKPDjNU7COd/MKJDQxkLp
HjANBgkqhkiG9w0BAQsFAAOCAQEAbkh5WOZ2ClOG2wA6WHqqW59nz7bhBPKWOgf9
McFD/74Fx8U7Pk50MK1cDH7s1rEOQDPiqZgAKhJXYm68Yzh/8abjL2t+lU8kyqgS
NNCgC6VDIWVSWBQIZAuEHtLZN3UWaN8/xbHJh0Uz7jxTSbCcHZIaXxP1G7W2dvW3
5QxNiQTeGD/ztVWRcMPBfI2Scm5XWOowmsOMtRRJRl9is1IeNXs0rTPRv1WJfU1f
rizX6JNDjXbHg1xChfSLPe+pdiafB3Fjr8s4//4RkzVsm78MJcuj1JfFfMzSN1hD
JXWSUic9TQbj0YIUvx2IvBB9zUu1CaoL9+IuFnH+ozEEYzs+GQ==
-----END CERTIFICATE-----
Run E2E tests: graph
Starting E2E functional tests ...
Parallelism requested for pytest is 2
============================= test session starts ==============================
platform linux -- Python 3.11.13, pytest-7.4.4, pluggy-1.5.0 -- /workspace/source/python/kserve/.venv/bin/python
cachedir: .pytest_cache
rootdir: /workspace/source/test/e2e
configfile: pytest.ini
plugins: asyncio-0.23.8, httpx-0.30.0, anyio-4.9.0, cov-5.0.0, xdist-3.6.1
asyncio: mode=Mode.STRICT
created: 2/2 workers
2 workers [11 items]
scheduling tests via WorkStealingScheduling
graph/test_inference_graph.py::test_inference_graph
graph/test_inference_graph.py::test_ig_scenario5 2026-04-16 18:38:43.598 2850 kserve INFO [conftest.py:configure_logger():40] Logger configured
2026-04-16 18:38:43.598 2853 kserve INFO [conftest.py:configure_logger():40] Logger configured
2026-04-16 18:38:43.600 2850 kserve.trace Starting test test_inference_graph
2026-04-16 18:38:43.600 2850 kserve.trace INFO [test_inference_graph.py:test_inference_graph():43] Starting test test_inference_graph
2026-04-16 18:38:43.601 2853 kserve.trace Starting test test_ig_scenario5
2026-04-16 18:38:43.601 2853 kserve.trace INFO [test_inference_graph.py:test_ig_scenario5():549] Starting test test_ig_scenario5
2026-04-16 18:38:43.602 2853 kserve.trace SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
2026-04-16 18:38:43.602 2853 kserve.trace INFO [test_inference_graph.py:setup_isvcs_for_test():207] SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
2026-04-16 18:38:43.602 2853 kserve.trace ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
2026-04-16 18:38:43.602 2853 kserve.trace INFO [test_inference_graph.py:setup_isvcs_for_test():208] ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
2026-04-16 18:38:43.603 2853 kserve.trace success_isvc_name is success-200-isvc-e1a28
2026-04-16 18:38:43.603 2853 kserve.trace INFO [test_inference_graph.py:test_ig_scenario5():554] success_isvc_name is success-200-isvc-e1a28
2026-04-16 18:38:43.603 2853 kserve.trace error_isvc_name is error-404-isvc-e1a28
2026-04-16 18:38:43.603 2853 kserve.trace INFO [test_inference_graph.py:test_ig_scenario5():555] error_isvc_name is error-404-isvc-e1a28
2026-04-16 18:40:13.952 2853 kserve.trace CI_USE_ISVC_HOST = 1
2026-04-16 18:40:13.952 2853 kserve.trace INFO [utils.py:get_isvc_endpoint():335] CI_USE_ISVC_HOST = 1
2026-04-16 18:40:13.952 2853 kserve.trace Host from isvc status URL = switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:40:13.952 2853 kserve.trace INFO [utils.py:get_isvc_endpoint():336] Host from isvc status URL = switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:40:13.952 2853 kserve.trace Network layer = istio
2026-04-16 18:40:13.952 2853 kserve.trace INFO [utils.py:get_isvc_endpoint():337] Network layer = istio
2026-04-16 18:40:13.952 2853 kserve.trace Using external route host: switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:40:13.952 2853 kserve.trace INFO [utils.py:get_isvc_endpoint():340] Using external route host: switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:40:13.952 2853 kserve.trace Sending Header = {'Host': 'switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com', 'Content-Type': 'application/json'}
2026-04-16 18:40:13.952 2853 kserve.trace INFO [utils.py:_predict():144] Sending Header = {'Host': 'switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com', 'Content-Type': 'application/json'}
2026-04-16 18:40:13.952 2853 kserve.trace base url = https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:40:13.952 2853 kserve.trace INFO [utils.py:_predict():145] base url = https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:40:13.953 2853 kserve.trace url: https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:40:13.953 2853 kserve.trace INFO [inference_client.py:infer():490] url: https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:40:13.953 2853 kserve.trace request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
2026-04-16 18:40:13.953 2853 kserve.trace INFO [inference_client.py:infer():491] request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
2026-04-16 18:41:03.934 2850 kserve.trace CI_USE_ISVC_HOST = 1
2026-04-16 18:41:03.934 2850 kserve.trace INFO [utils.py:get_isvc_endpoint():335] CI_USE_ISVC_HOST = 1
2026-04-16 18:41:03.934 2850 kserve.trace Host from isvc status URL = model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:41:03.934 2850 kserve.trace INFO [utils.py:get_isvc_endpoint():336] Host from isvc status URL = model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:41:03.934 2850 kserve.trace Network layer = istio
2026-04-16 18:41:03.934 2850 kserve.trace INFO [utils.py:get_isvc_endpoint():337] Network layer = istio
2026-04-16 18:41:03.934 2850 kserve.trace Using external route host: model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:41:03.934 2850 kserve.trace INFO [utils.py:get_isvc_endpoint():340] Using external route host: model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:41:03.934 2850 kserve.trace Sending Header = {'Host': 'model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com', 'Content-Type': 'application/json'}
2026-04-16 18:41:03.934 2850 kserve.trace INFO [utils.py:_predict():144] Sending Header = {'Host': 'model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com', 'Content-Type': 'application/json'}
2026-04-16 18:41:03.934 2850 kserve.trace base url = https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:41:03.934 2850 kserve.trace INFO [utils.py:_predict():145] base url = https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:41:03.934 2850 kserve.trace url: https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:41:03.934 2850 kserve.trace INFO [inference_client.py:infer():490] url: https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:41:03.935 2850 kserve.trace request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 18:41:03.935 2850 kserve.trace INFO [inference_client.py:infer():491] request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 18:48:18.574 2853 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 18:48:18.574 2853 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 18:48:18.575 2853 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 18:48:18.575 2853 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 18:48:18.575 2853 kserve.trace Transient error on attempt 1/5, retrying in 5s...
2026-04-16 18:48:18.575 2853 kserve.trace INFO [utils.py:_predict():166] Transient error on attempt 1/5, retrying in 5s...
2026-04-16 18:48:23.580 2853 kserve.trace url: https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:48:23.580 2853 kserve.trace INFO [inference_client.py:infer():490] url: https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:48:23.580 2853 kserve.trace request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
2026-04-16 18:48:23.580 2853 kserve.trace INFO [inference_client.py:infer():491] request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
2026-04-16 18:49:08.468 2850 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 18:49:08.468 2850 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 18:49:08.468 2850 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 18:49:08.468 2850 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 18:49:08.468 2850 kserve.trace Transient error on attempt 1/5, retrying in 5s...
2026-04-16 18:49:08.468 2850 kserve.trace INFO [utils.py:_predict():166] Transient error on attempt 1/5, retrying in 5s...
2026-04-16 18:49:13.473 2850 kserve.trace url: https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:49:13.473 2850 kserve.trace INFO [inference_client.py:infer():490] url: https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:49:13.473 2850 kserve.trace request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 18:49:13.473 2850 kserve.trace INFO [inference_client.py:infer():491] request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 18:56:28.136 2853 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 18:56:28.136 2853 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 18:56:28.136 2853 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 18:56:28.136 2853 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 18:56:28.136 2853 kserve.trace Transient error on attempt 2/5, retrying in 5s...
2026-04-16 18:56:28.136 2853 kserve.trace INFO [utils.py:_predict():166] Transient error on attempt 2/5, retrying in 5s...
2026-04-16 18:56:33.141 2853 kserve.trace url: https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:56:33.141 2853 kserve.trace INFO [inference_client.py:infer():490] url: https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:56:33.141 2853 kserve.trace request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
2026-04-16 18:56:33.141 2853 kserve.trace INFO [inference_client.py:infer():491] request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
2026-04-16 18:57:17.858 2850 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 18:57:17.858 2850 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 18:57:17.858 2850 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 18:57:17.858 2850 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 18:57:17.858 2850 kserve.trace Transient error on attempt 2/5, retrying in 5s...
2026-04-16 18:57:17.858 2850 kserve.trace INFO [utils.py:_predict():166] Transient error on attempt 2/5, retrying in 5s...
2026-04-16 18:57:22.863 2850 kserve.trace url: https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:57:22.863 2850 kserve.trace INFO [inference_client.py:infer():490] url: https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 18:57:22.864 2850 kserve.trace request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 18:57:22.864 2850 kserve.trace INFO [inference_client.py:infer():491] request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 19:04:37.675 2853 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:04:37.675 2853 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:04:37.675 2853 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:04:37.675 2853 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:04:37.675 2853 kserve.trace Transient error on attempt 3/5, retrying in 5s...
2026-04-16 19:04:37.675 2853 kserve.trace INFO [utils.py:_predict():166] Transient error on attempt 3/5, retrying in 5s...
2026-04-16 19:04:42.681 2853 kserve.trace url: https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:04:42.681 2853 kserve.trace INFO [inference_client.py:infer():490] url: https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:04:42.681 2853 kserve.trace request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
2026-04-16 19:04:42.681 2853 kserve.trace INFO [inference_client.py:infer():491] request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
2026-04-16 19:05:27.352 2850 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:05:27.352 2850 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:05:27.353 2850 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:05:27.353 2850 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:05:27.353 2850 kserve.trace Transient error on attempt 3/5, retrying in 5s...
2026-04-16 19:05:27.353 2850 kserve.trace INFO [utils.py:_predict():166] Transient error on attempt 3/5, retrying in 5s...
2026-04-16 19:05:32.358 2850 kserve.trace url: https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:05:32.358 2850 kserve.trace INFO [inference_client.py:infer():490] url: https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:05:32.358 2850 kserve.trace request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 19:05:32.358 2850 kserve.trace INFO [inference_client.py:infer():491] request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 19:12:47.230 2853 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:12:47.230 2853 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:12:47.230 2853 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:12:47.230 2853 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:12:47.231 2853 kserve.trace Transient error on attempt 4/5, retrying in 5s...
2026-04-16 19:12:47.231 2853 kserve.trace INFO [utils.py:_predict():166] Transient error on attempt 4/5, retrying in 5s...
2026-04-16 19:12:52.236 2853 kserve.trace url: https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:12:52.236 2853 kserve.trace INFO [inference_client.py:infer():490] url: https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:12:52.236 2853 kserve.trace request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
2026-04-16 19:12:52.236 2853 kserve.trace INFO [inference_client.py:infer():491] request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
2026-04-16 19:13:36.799 2850 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:13:36.799 2850 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:13:36.799 2850 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:13:36.799 2850 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:13:36.800 2850 kserve.trace Transient error on attempt 4/5, retrying in 5s...
2026-04-16 19:13:36.800 2850 kserve.trace INFO [utils.py:_predict():166] Transient error on attempt 4/5, retrying in 5s...
2026-04-16 19:13:41.805 2850 kserve.trace url: https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:13:41.805 2850 kserve.trace INFO [inference_client.py:infer():490] url: https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:13:41.805 2850 kserve.trace request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 19:13:41.805 2850 kserve.trace INFO [inference_client.py:infer():491] request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 19:20:56.851 2853 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:20:56.851 2853 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:20:56.851 2853 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:20:56.851 2853 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:20:57.020 2853 kserve.trace Starting test test_ig_scenario6
2026-04-16 19:20:57.020 2853 kserve.trace INFO [test_inference_graph.py:test_ig_scenario6():613] Starting test test_ig_scenario6
2026-04-16 19:20:57.021 2853 kserve.trace SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
2026-04-16 19:20:57.021 2853 kserve.trace INFO [test_inference_graph.py:setup_isvcs_for_test():207] SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
2026-04-16 19:20:57.021 2853 kserve.trace ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
2026-04-16 19:20:57.021 2853 kserve.trace INFO [test_inference_graph.py:setup_isvcs_for_test():208] ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
2026-04-16 19:20:57.021 2853 kserve.trace success_isvc_name is success-200-isvc-4f739
2026-04-16 19:20:57.021 2853 kserve.trace INFO [test_inference_graph.py:test_ig_scenario6():618] success_isvc_name is success-200-isvc-4f739
2026-04-16 19:20:57.021 2853 kserve.trace error_isvc_name is error-404-isvc-4f739
2026-04-16 19:20:57.021 2853 kserve.trace INFO [test_inference_graph.py:test_ig_scenario6():619] error_isvc_name is error-404-isvc-4f739
[gw1] FAILED graph/test_inference_graph.py::test_ig_scenario5
graph/test_inference_graph.py::test_ig_scenario6 2026-04-16 19:21:46.404 2850 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:21:46.404 2850 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:21:46.404 2850 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:21:46.404 2850 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:21:46.630 2850 kserve.trace Starting test test_ig_scenario1
2026-04-16 19:21:46.630 2850 kserve.trace INFO [test_inference_graph.py:test_ig_scenario1():239] Starting test test_ig_scenario1
2026-04-16 19:21:46.630 2850 kserve.trace SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
2026-04-16 19:21:46.630 2850 kserve.trace INFO [test_inference_graph.py:setup_isvcs_for_test():207] SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
2026-04-16 19:21:46.630 2850 kserve.trace ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
2026-04-16 19:21:46.630 2850 kserve.trace INFO [test_inference_graph.py:setup_isvcs_for_test():208] ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
[gw0] FAILED graph/test_inference_graph.py::test_inference_graph
graph/test_inference_graph.py::test_ig_scenario1 2026-04-16 19:21:46.631 2850 kserve.trace success_isvc_name is success-200-isvc-ed38c
2026-04-16 19:21:46.631 2850 kserve.trace INFO [test_inference_graph.py:test_ig_scenario1():244] success_isvc_name is success-200-isvc-ed38c
2026-04-16 19:21:46.631 2850 kserve.trace error_isvc_name is error-404-isvc-ed38c
2026-04-16 19:21:46.631 2850 kserve.trace INFO [test_inference_graph.py:test_ig_scenario1():245] error_isvc_name is error-404-isvc-ed38c
2026-04-16 19:22:17.325 2853 kserve.trace CI_USE_ISVC_HOST = 1
2026-04-16 19:22:17.325 2853 kserve.trace INFO [utils.py:get_isvc_endpoint():335] CI_USE_ISVC_HOST = 1
2026-04-16 19:22:17.325 2853 kserve.trace Host from isvc status URL = switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:22:17.325 2853 kserve.trace INFO [utils.py:get_isvc_endpoint():336] Host from isvc status URL = switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:22:17.325 2853 kserve.trace Network layer = istio
2026-04-16 19:22:17.325 2853 kserve.trace INFO [utils.py:get_isvc_endpoint():337] Network layer = istio
2026-04-16 19:22:17.326 2853 kserve.trace Using external route host: switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:22:17.326 2853 kserve.trace INFO [utils.py:get_isvc_endpoint():340] Using external route host: switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:22:17.326 2853 kserve.trace Sending Header = {'Host': 'switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com', 'Content-Type': 'application/json'}
2026-04-16 19:22:17.326 2853 kserve.trace INFO [utils.py:_predict():144] Sending Header = {'Host': 'switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com', 'Content-Type': 'application/json'}
2026-04-16 19:22:17.326 2853 kserve.trace base url = https://switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:22:17.326 2853 kserve.trace INFO [utils.py:_predict():145] base url = https://switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:22:17.326 2853 kserve.trace url: https://switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:22:17.326 2853 kserve.trace INFO [inference_client.py:infer():490] url: https://switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:22:17.326 2853 kserve.trace request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
2026-04-16 19:22:17.326 2853 kserve.trace INFO [inference_client.py:infer():491] request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
2026-04-16 19:23:06.817 2850 kserve.trace CI_USE_ISVC_HOST = 1
2026-04-16 19:23:06.817 2850 kserve.trace INFO [utils.py:get_isvc_endpoint():335] CI_USE_ISVC_HOST = 1
2026-04-16 19:23:06.817 2850 kserve.trace Host from isvc status URL = sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:23:06.817 2850 kserve.trace INFO [utils.py:get_isvc_endpoint():336] Host from isvc status URL = sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:23:06.817 2850 kserve.trace Network layer = istio
2026-04-16 19:23:06.817 2850 kserve.trace INFO [utils.py:get_isvc_endpoint():337] Network layer = istio
2026-04-16 19:23:06.817 2850 kserve.trace Using external route host: sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:23:06.817 2850 kserve.trace INFO [utils.py:get_isvc_endpoint():340] Using external route host: sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:23:06.817 2850 kserve.trace Sending Header = {'Host': 'sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com', 'Content-Type': 'application/json'}
2026-04-16 19:23:06.817 2850 kserve.trace INFO [utils.py:_predict():144] Sending Header = {'Host': 'sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com', 'Content-Type': 'application/json'}
2026-04-16 19:23:06.817 2850 kserve.trace base url = https://sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:23:06.817 2850 kserve.trace INFO [utils.py:_predict():145] base url = https://sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:23:06.817 2850 kserve.trace url: https://sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:23:06.817 2850 kserve.trace INFO [inference_client.py:infer():490] url: https://sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:23:06.818 2850 kserve.trace request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 19:23:06.818 2850 kserve.trace INFO [inference_client.py:infer():491] request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 19:30:21.817 2853 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:30:21.817 2853 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:30:21.817 2853 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:30:21.817 2853 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:30:21.817 2853 kserve.trace Transient error on attempt 1/5, retrying in 5s...
2026-04-16 19:30:21.817 2853 kserve.trace INFO [utils.py:_predict():166] Transient error on attempt 1/5, retrying in 5s...
2026-04-16 19:30:26.822 2853 kserve.trace url: https://switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:30:26.822 2853 kserve.trace INFO [inference_client.py:infer():490] url: https://switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:30:26.823 2853 kserve.trace request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
2026-04-16 19:30:26.823 2853 kserve.trace INFO [inference_client.py:infer():491] request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
2026-04-16 19:31:11.347 2850 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:31:11.347 2850 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:31:11.347 2850 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:31:11.347 2850 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:31:11.347 2850 kserve.trace Transient error on attempt 1/5, retrying in 5s...
2026-04-16 19:31:11.347 2850 kserve.trace INFO [utils.py:_predict():166] Transient error on attempt 1/5, retrying in 5s...
2026-04-16 19:31:16.352 2850 kserve.trace url: https://sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:31:16.352 2850 kserve.trace INFO [inference_client.py:infer():490] url: https://sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:31:16.352 2850 kserve.trace request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 19:31:16.352 2850 kserve.trace INFO [inference_client.py:infer():491] request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 19:38:31.334 2853 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:38:31.334 2853 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:38:31.334 2853 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:38:31.334 2853 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:38:31.334 2853 kserve.trace Transient error on attempt 2/5, retrying in 5s...
2026-04-16 19:38:31.334 2853 kserve.trace INFO [utils.py:_predict():166] Transient error on attempt 2/5, retrying in 5s...
2026-04-16 19:38:36.339 2853 kserve.trace url: https://switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:38:36.339 2853 kserve.trace INFO [inference_client.py:infer():490] url: https://switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:38:36.340 2853 kserve.trace request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
2026-04-16 19:38:36.340 2853 kserve.trace INFO [inference_client.py:infer():491] request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
2026-04-16 19:39:20.938 2850 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:39:20.938 2850 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:39:20.938 2850 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:39:20.938 2850 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:39:20.938 2850 kserve.trace Transient error on attempt 2/5, retrying in 5s...
2026-04-16 19:39:20.938 2850 kserve.trace INFO [utils.py:_predict():166] Transient error on attempt 2/5, retrying in 5s...
2026-04-16 19:39:25.943 2850 kserve.trace url: https://sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:39:25.943 2850 kserve.trace INFO [inference_client.py:infer():490] url: https://sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:39:25.944 2850 kserve.trace request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 19:39:25.944 2850 kserve.trace INFO [inference_client.py:infer():491] request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 19:46:40.860 2853 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:46:40.860 2853 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:46:40.860 2853 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:46:40.860 2853 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:46:40.860 2853 kserve.trace Transient error on attempt 3/5, retrying in 5s...
2026-04-16 19:46:40.860 2853 kserve.trace INFO [utils.py:_predict():166] Transient error on attempt 3/5, retrying in 5s...
2026-04-16 19:46:45.866 2853 kserve.trace url: https://switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:46:45.866 2853 kserve.trace INFO [inference_client.py:infer():490] url: https://switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:46:45.866 2853 kserve.trace request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
2026-04-16 19:46:45.866 2853 kserve.trace INFO [inference_client.py:infer():491] request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
2026-04-16 19:47:30.485 2850 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:47:30.485 2850 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:47:30.486 2850 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:47:30.486 2850 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:47:30.486 2850 kserve.trace Transient error on attempt 3/5, retrying in 5s...
2026-04-16 19:47:30.486 2850 kserve.trace INFO [utils.py:_predict():166] Transient error on attempt 3/5, retrying in 5s...
2026-04-16 19:47:35.491 2850 kserve.trace url: https://sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:47:35.491 2850 kserve.trace INFO [inference_client.py:infer():490] url: https://sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:47:35.491 2850 kserve.trace request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 19:47:35.491 2850 kserve.trace INFO [inference_client.py:infer():491] request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 19:54:50.297 2853 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:54:50.297 2853 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:54:50.298 2853 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:54:50.298 2853 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:54:50.298 2853 kserve.trace Transient error on attempt 4/5, retrying in 5s...
2026-04-16 19:54:50.298 2853 kserve.trace INFO [utils.py:_predict():166] Transient error on attempt 4/5, retrying in 5s...
2026-04-16 19:54:55.303 2853 kserve.trace url: https://switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:54:55.303 2853 kserve.trace INFO [inference_client.py:infer():490] url: https://switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:54:55.303 2853 kserve.trace request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
2026-04-16 19:54:55.303 2853 kserve.trace INFO [inference_client.py:infer():491] request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
2026-04-16 19:55:40.056 2850 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:55:40.056 2850 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:55:40.056 2850 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:55:40.056 2850 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 19:55:40.056 2850 kserve.trace Transient error on attempt 4/5, retrying in 5s...
2026-04-16 19:55:40.056 2850 kserve.trace INFO [utils.py:_predict():166] Transient error on attempt 4/5, retrying in 5s...
2026-04-16 19:55:45.062 2850 kserve.trace url: https://sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:55:45.062 2850 kserve.trace INFO [inference_client.py:infer():490] url: https://sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 19:55:45.062 2850 kserve.trace request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 19:55:45.062 2850 kserve.trace INFO [inference_client.py:infer():491] request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 20:02:59.793 2853 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 20:02:59.793 2853 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 20:02:59.793 2853 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 20:02:59.793 2853 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 20:02:59.824 2853 kserve.trace Starting test test_ig_scenario7
2026-04-16 20:02:59.824 2853 kserve.trace INFO [test_inference_graph.py:test_ig_scenario7():680] Starting test test_ig_scenario7
2026-04-16 20:02:59.824 2853 kserve.trace SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
2026-04-16 20:02:59.824 2853 kserve.trace INFO [test_inference_graph.py:setup_isvcs_for_test():207] SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
2026-04-16 20:02:59.824 2853 kserve.trace ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
2026-04-16 20:02:59.824 2853 kserve.trace INFO [test_inference_graph.py:setup_isvcs_for_test():208] ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
[gw1] FAILED graph/test_inference_graph.py::test_ig_scenario6
graph/test_inference_graph.py::test_ig_scenario7 2026-04-16 20:02:59.825 2853 kserve.trace success_isvc_name is success-200-isvc-42610
2026-04-16 20:02:59.825 2853 kserve.trace INFO [test_inference_graph.py:test_ig_scenario7():685] success_isvc_name is success-200-isvc-42610
2026-04-16 20:02:59.825 2853 kserve.trace error_isvc_name is error-404-isvc-42610
2026-04-16 20:02:59.825 2853 kserve.trace INFO [test_inference_graph.py:test_ig_scenario7():686] error_isvc_name is error-404-isvc-42610
2026-04-16 20:03:49.611 2850 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 20:03:49.611 2850 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 20:03:49.611 2850 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 20:03:49.611 2850 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
[gw0] FAILED graph/test_inference_graph.py::test_ig_scenario1 2026-04-16 20:03:49.638 2850 kserve.trace Starting test test_ig_scenario2
2026-04-16 20:03:49.638 2850 kserve.trace INFO [test_inference_graph.py:test_ig_scenario2():314] Starting test test_ig_scenario2
2026-04-16 20:03:49.638 2850 kserve.trace SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
2026-04-16 20:03:49.638 2850 kserve.trace INFO [test_inference_graph.py:setup_isvcs_for_test():207] SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
2026-04-16 20:03:49.638 2850 kserve.trace ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
2026-04-16 20:03:49.638 2850 kserve.trace INFO [test_inference_graph.py:setup_isvcs_for_test():208] ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
graph/test_inference_graph.py::test_ig_scenario2 2026-04-16 20:03:49.639 2850 kserve.trace success_isvc_name is success-200-isvc-b1781
2026-04-16 20:03:49.639 2850 kserve.trace INFO [test_inference_graph.py:test_ig_scenario2():319] success_isvc_name is success-200-isvc-b1781
2026-04-16 20:03:49.639 2850 kserve.trace error_isvc_name is error-404-isvc-b1781
2026-04-16 20:03:49.639 2850 kserve.trace INFO [test_inference_graph.py:test_ig_scenario2():320] error_isvc_name is error-404-isvc-b1781
2026-04-16 20:04:20.115 2853 kserve.trace CI_USE_ISVC_HOST = 1
2026-04-16 20:04:20.115 2853 kserve.trace INFO [utils.py:get_isvc_endpoint():335] CI_USE_ISVC_HOST = 1
2026-04-16 20:04:20.116 2853 kserve.trace Host from isvc status URL = ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:04:20.116 2853 kserve.trace INFO [utils.py:get_isvc_endpoint():336] Host from isvc status URL = ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:04:20.116 2853 kserve.trace Network layer = istio
2026-04-16 20:04:20.116 2853 kserve.trace INFO [utils.py:get_isvc_endpoint():337] Network layer = istio
2026-04-16 20:04:20.116 2853 kserve.trace Using external route host: ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:04:20.116 2853 kserve.trace INFO [utils.py:get_isvc_endpoint():340] Using external route host: ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:04:20.116 2853 kserve.trace Sending Header = {'Host': 'ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com', 'Content-Type': 'application/json'}
2026-04-16 20:04:20.116 2853 kserve.trace INFO [utils.py:_predict():144] Sending Header = {'Host': 'ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com', 'Content-Type': 'application/json'}
2026-04-16 20:04:20.116 2853 kserve.trace base url = https://ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:04:20.116 2853 kserve.trace INFO [utils.py:_predict():145] base url = https://ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:04:20.116 2853 kserve.trace url: https://ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:04:20.116 2853 kserve.trace INFO [inference_client.py:infer():490] url: https://ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:04:20.116 2853 kserve.trace request data: {'decision_picker': 'SUCCESS', 'key1': {'name': 'abc'}}
2026-04-16 20:04:20.116 2853 kserve.trace INFO [inference_client.py:infer():491] request data: {'decision_picker': 'SUCCESS', 'key1': {'name': 'abc'}}
2026-04-16 20:05:09.819 2850 kserve.trace CI_USE_ISVC_HOST = 1
2026-04-16 20:05:09.819 2850 kserve.trace INFO [utils.py:get_isvc_endpoint():335] CI_USE_ISVC_HOST = 1
2026-04-16 20:05:09.819 2850 kserve.trace Host from isvc status URL = sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:05:09.819 2850 kserve.trace INFO [utils.py:get_isvc_endpoint():336] Host from isvc status URL = sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:05:09.819 2850 kserve.trace Network layer = istio
2026-04-16 20:05:09.819 2850 kserve.trace INFO [utils.py:get_isvc_endpoint():337] Network layer = istio
2026-04-16 20:05:09.819 2850 kserve.trace Using external route host: sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:05:09.819 2850 kserve.trace INFO [utils.py:get_isvc_endpoint():340] Using external route host: sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:05:09.820 2850 kserve.trace Sending Header = {'Host': 'sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com', 'Content-Type': 'application/json'}
2026-04-16 20:05:09.820 2850 kserve.trace INFO [utils.py:_predict():144] Sending Header = {'Host': 'sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com', 'Content-Type': 'application/json'}
2026-04-16 20:05:09.820 2850 kserve.trace base url = https://sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:05:09.820 2850 kserve.trace INFO [utils.py:_predict():145] base url = https://sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:05:09.820 2850 kserve.trace url: https://sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:05:09.820 2850 kserve.trace INFO [inference_client.py:infer():490] url: https://sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:05:09.820 2850 kserve.trace request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 20:05:09.820 2850 kserve.trace INFO [inference_client.py:infer():491] request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 20:12:24.623 2853 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 20:12:24.623 2853 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 20:12:24.623 2853 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 20:12:24.623 2853 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 20:12:24.624 2853 kserve.trace Transient error on attempt 1/5, retrying in 5s...
2026-04-16 20:12:24.624 2853 kserve.trace INFO [utils.py:_predict():166] Transient error on attempt 1/5, retrying in 5s...
2026-04-16 20:12:29.628 2853 kserve.trace url: https://ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:12:29.628 2853 kserve.trace INFO [inference_client.py:infer():490] url: https://ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:12:29.628 2853 kserve.trace request data: {'decision_picker': 'SUCCESS', 'key1': {'name': 'abc'}}
2026-04-16 20:12:29.628 2853 kserve.trace INFO [inference_client.py:infer():491] request data: {'decision_picker': 'SUCCESS', 'key1': {'name': 'abc'}}
2026-04-16 20:13:14.391 2850 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 20:13:14.391 2850 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 20:13:14.391 2850 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 20:13:14.391 2850 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 20:13:14.391 2850 kserve.trace Transient error on attempt 1/5, retrying in 5s...
2026-04-16 20:13:14.391 2850 kserve.trace INFO [utils.py:_predict():166] Transient error on attempt 1/5, retrying in 5s...
2026-04-16 20:13:19.396 2850 kserve.trace url: https://sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:13:19.396 2850 kserve.trace INFO [inference_client.py:infer():490] url: https://sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:13:19.397 2850 kserve.trace request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 20:13:19.397 2850 kserve.trace INFO [inference_client.py:infer():491] request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 20:20:34.107 2853 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 20:20:34.107 2853 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 20:20:34.107 2853 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 20:20:34.107 2853 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 20:20:34.107 2853 kserve.trace Transient error on attempt 2/5, retrying in 5s...
2026-04-16 20:20:34.107 2853 kserve.trace INFO [utils.py:_predict():166] Transient error on attempt 2/5, retrying in 5s...
2026-04-16 20:20:39.112 2853 kserve.trace url: https://ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:20:39.112 2853 kserve.trace INFO [inference_client.py:infer():490] url: https://ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:20:39.112 2853 kserve.trace request data: {'decision_picker': 'SUCCESS', 'key1': {'name': 'abc'}}
2026-04-16 20:20:39.112 2853 kserve.trace INFO [inference_client.py:infer():491] request data: {'decision_picker': 'SUCCESS', 'key1': {'name': 'abc'}}
2026-04-16 20:21:23.965 2850 kserve.trace response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 20:21:23.965 2850 kserve.trace INFO [inference_client.py:infer():507] response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 20:21:23.966 2850 kserve.trace HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 20:21:23.966 2850 kserve.trace INFO [utils.py:_predict():156] HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
2026-04-16 20:21:23.966 2850 kserve.trace Transient error on attempt 2/5, retrying in 5s...
2026-04-16 20:21:23.966 2850 kserve.trace INFO [utils.py:_predict():166] Transient error on attempt 2/5, retrying in 5s...
2026-04-16 20:21:28.971 2850 kserve.trace url: https://sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:21:28.971 2850 kserve.trace INFO [inference_client.py:infer():490] url: https://sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
2026-04-16 20:21:28.971 2850 kserve.trace request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 20:21:28.971 2850 kserve.trace INFO [inference_client.py:infer():491] request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
2026-04-16 20:28:43.832 2853 kserve.trace Starting test test_ig_scenario8
2026-04-16 20:28:43.832 2853 kserve.trace INFO [test_inference_graph.py:test_ig_scenario8():747] Starting test test_ig_scenario8
2026-04-16 20:28:43.832 2853 kserve.trace SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
2026-04-16 20:28:43.832 2853 kserve.trace INFO [test_inference_graph.py:setup_isvcs_for_test():207] SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
2026-04-16 20:28:43.832 2853 kserve.trace ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
2026-04-16 20:28:43.832 2853 kserve.trace INFO [test_inference_graph.py:setup_isvcs_for_test():208] ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
2026-04-16 20:28:43.833 2853 kserve.trace success_isvc_name is success-200-isvc-071c5
2026-04-16 20:28:43.833 2853 kserve.trace INFO [test_inference_graph.py:test_ig_scenario8():752] success_isvc_name is success-200-isvc-071c5
2026-04-16 20:28:43.833 2853 kserve.trace error_isvc_name is error-404-isvc-071c5
2026-04-16 20:28:43.833 2853 kserve.trace INFO [test_inference_graph.py:test_ig_scenario8():753] error_isvc_name is error-404-isvc-071c5
[gw1] FAILED graph/test_inference_graph.py::test_ig_scenario7
graph/test_inference_graph.py::test_ig_scenario8 2026-04-16 20:28:43.968 2853 kserve.trace Starting test test_ig_scenario9
2026-04-16 20:28:43.968 2853 kserve.trace INFO [test_inference_graph.py:test_ig_scenario9():813] Starting test test_ig_scenario9
2026-04-16 20:28:43.968 2853 kserve.trace SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
2026-04-16 20:28:43.968 2853 kserve.trace INFO [test_inference_graph.py:setup_isvcs_for_test():207] SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
2026-04-16 20:28:43.969 2853 kserve.trace ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
2026-04-16 20:28:43.969 2853 kserve.trace INFO [test_inference_graph.py:setup_isvcs_for_test():208] ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
[gw1] FAILED graph/test_inference_graph.py::test_ig_scenario8
graph/test_inference_graph.py::test_ig_scenario9 2026-04-16 20:28:43.969 2853 kserve.trace success_isvc_name is success-200-isvc-0f5dd
2026-04-16 20:28:43.969 2853 kserve.trace INFO [test_inference_graph.py:test_ig_scenario9():818] success_isvc_name is success-200-isvc-0f5dd
2026-04-16 20:28:43.969 2853 kserve.trace error_isvc_name is error-404-isvc-0f5dd
2026-04-16 20:28:43.969 2853 kserve.trace INFO [test_inference_graph.py:test_ig_scenario9():819] error_isvc_name is error-404-isvc-0f5dd
[gw1] FAILED graph/test_inference_graph.py::test_ig_scenario9
graph/test_inference_graph.py::test_ig_scenario10 2026-04-16 20:28:44.189 2853 kserve.trace Starting test test_ig_scenario10
2026-04-16 20:28:44.189 2853 kserve.trace INFO [test_inference_graph.py:test_ig_scenario10():875] Starting test test_ig_scenario10
2026-04-16 20:28:44.190 2853 kserve.trace SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
2026-04-16 20:28:44.190 2853 kserve.trace INFO [test_inference_graph.py:setup_isvcs_for_test():207] SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
2026-04-16 20:28:44.190 2853 kserve.trace ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
2026-04-16 20:28:44.190 2853 kserve.trace INFO [test_inference_graph.py:setup_isvcs_for_test():208] ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
2026-04-16 20:28:44.190 2853 kserve.trace success_isvc_name is success-200-isvc-c640d
2026-04-16 20:28:44.190 2853 kserve.trace INFO [test_inference_graph.py:test_ig_scenario10():880] success_isvc_name is success-200-isvc-c640d
2026-04-16 20:28:44.190 2853 kserve.trace error_isvc_name is error-404-isvc-c640d
2026-04-16 20:28:44.190 2853 kserve.trace INFO [test_inference_graph.py:test_ig_scenario10():881] error_isvc_name is error-404-isvc-c640d
[gw1] FAILED graph/test_inference_graph.py::test_ig_scenario10
graph/test_inference_graph.py::test_ig_scenario4 2026-04-16 20:28:44.320 2853 kserve.trace Starting test test_ig_scenario4
2026-04-16 20:28:44.320 2853 kserve.trace INFO [test_inference_graph.py:test_ig_scenario4():456] Starting test test_ig_scenario4
2026-04-16 20:28:44.320 2853 kserve.trace SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
2026-04-16 20:28:44.320 2853 kserve.trace INFO [test_inference_graph.py:setup_isvcs_for_test():207] SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
2026-04-16 20:28:44.320 2853 kserve.trace ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
2026-04-16 20:28:44.320 2853 kserve.trace INFO [test_inference_graph.py:setup_isvcs_for_test():208] ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
2026-04-16 20:28:44.321 2853 kserve.trace success_isvc_name is success-200-isvc-d7402
2026-04-16 20:28:44.321 2853 kserve.trace INFO [test_inference_graph.py:test_ig_scenario4():461] success_isvc_name is success-200-isvc-d7402
2026-04-16 20:28:44.321 2853 kserve.trace error_isvc_name is error-404-isvc-d7402
2026-04-16 20:28:44.321 2853 kserve.trace INFO [test_inference_graph.py:test_ig_scenario4():462] error_isvc_name is error-404-isvc-d7402
[gw1] FAILED graph/test_inference_graph.py::test_ig_scenario4
[gw0] FAILED graph/test_inference_graph.py::test_ig_scenario2
graph/test_inference_graph.py::test_ig_scenario3 2026-04-16 20:29:33.608 2850 kserve.trace Starting test test_ig_scenario3
2026-04-16 20:29:33.608 2850 kserve.trace INFO [test_inference_graph.py:test_ig_scenario3():382] Starting test test_ig_scenario3
2026-04-16 20:29:33.608 2850 kserve.trace SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
2026-04-16 20:29:33.608 2850 kserve.trace INFO [test_inference_graph.py:setup_isvcs_for_test():207] SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
2026-04-16 20:29:33.609 2850 kserve.trace ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
2026-04-16 20:29:33.609 2850 kserve.trace INFO [test_inference_graph.py:setup_isvcs_for_test():208] ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
2026-04-16 20:29:33.609 2850 kserve.trace success_isvc_name is success-200-isvc-aeddd
2026-04-16 20:29:33.609 2850 kserve.trace INFO [test_inference_graph.py:test_ig_scenario3():387] success_isvc_name is success-200-isvc-aeddd
2026-04-16 20:29:33.609 2850 kserve.trace error_isvc_name is error-404-isvc-aeddd
2026-04-16 20:29:33.609 2850 kserve.trace INFO [test_inference_graph.py:test_ig_scenario3():388] error_isvc_name is error-404-isvc-aeddd
[gw0] FAILED graph/test_inference_graph.py::test_ig_scenario3
=================================== FAILURES ===================================
______________________________ test_ig_scenario5 _______________________________
[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python
rest_v1_client =
@pytest.mark.graph
@pytest.mark.kourier
@pytest.mark.asyncio(scope="session")
async def test_ig_scenario5(rest_v1_client):
"""
Scenario: Switch graph where a match would happen for error node and then error would return but IG will continue
execution and call the next step in the flow as error step will be a soft dependency.
Expectation: IG will return response of success_isvc.
"""
logger.info("Starting test test_ig_scenario5")
suffix = str(uuid.uuid4())[1:6]
success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test(
suffix
)
logger.info(f"success_isvc_name is {success_isvc_name}")
logger.info(f"error_isvc_name is {error_isvc_name}")
kserve_client = KServeClient(
config_file=os.environ.get("KUBECONFIG", "~/.kube/config")
)
kserve_client.create(success_isvc)
kserve_client.create(error_isvc)
# Create graph
graph_name = "-".join(["switch-graph", suffix])
# Because we run from test/e2e location in run-e2e-tests.sh
deployment_yaml_path = os.path.join(
IG_TEST_RESOURCES_BASE_LOCATION, "ig_test_switch_scenario_5.yaml"
)
# Read YAML file
with open(deployment_yaml_path, "r") as stream:
file_content = stream.read()
resource_template = Template(file_content)
substitutions = {
"graph_name": graph_name,
"error_404_isvc_id": error_isvc_name,
"success_200_isvc_id": success_isvc_name,
}
resource_body_after_rendering = yaml.safe_load(
resource_template.render(substitutions)
)
kserve_client.wait_isvc_ready(success_isvc_name, namespace=KSERVE_TEST_NAMESPACE)
kserve_client.wait_isvc_ready(error_isvc_name, namespace=KSERVE_TEST_NAMESPACE)
create_ig_using_custom_object_api(resource_body_after_rendering)
kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE)
> response = await predict_ig(
rest_v1_client,
graph_name,
os.path.join(
IG_TEST_RESOURCES_BASE_LOCATION, "switch_call_error_picker_input.json"
),
)
graph/test_inference_graph.py:590:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
client =
ig_name = 'switch-graph-e1a28'
input_path = 'graph/test-resources/switch_call_error_picker_input.json'
version = 'v1alpha1', network_layer = 'istio'
async def predict_ig(
client: InferenceRESTClient,
ig_name,
input_path,
version=constants.KSERVE_V1ALPHA1_VERSION,
network_layer: str = "istio",
) -> Union[InferResponse, Dict]:
kserve_client = KServeClient(
config_file=os.environ.get("KUBECONFIG", "~/.kube/config")
)
ig = kserve_client.get_inference_graph(
ig_name,
namespace=KSERVE_TEST_NAMESPACE,
version=version,
)
scheme, cluster_ip, host, _ = get_isvc_endpoint(ig, network_layer)
url = f"{scheme}://{cluster_ip}"
> return await predict(client, url, host, input_path, is_graph=True)
common/utils.py:193:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
client =
url = 'https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
host = 'switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
input = 'graph/test-resources/switch_call_error_picker_input.json'
model_name = None, is_batch = False, is_graph = True
async def predict(
client: InferenceRESTClient,
url,
host,
input: Union[str, InferRequest],
model_name=None,
is_batch=False,
is_graph=False,
) -> Union[InferResponse, Dict, List[Union[Dict, InferResponse]]]:
if isinstance(input, str):
with open(input) as json_file:
data = json.load(json_file)
else:
data = input
headers = {"Host": host, "Content-Type": "application/json"}
if is_batch:
with futures.ThreadPoolExecutor(max_workers=4) as executor:
loop = asyncio.get_running_loop()
future_list = [
await loop.run_in_executor(
executor,
_predict,
client,
url,
input_data,
model_name,
headers,
is_graph,
)
for input_data in data
]
result = await asyncio.gather(*future_list)
else:
> result = await _predict(
client,
url,
data,
model_name=model_name,
headers=headers,
is_graph=is_graph,
)
common/utils.py:120:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
client =
url = 'https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
input_data = {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
model_name = None
headers = {'Content-Type': 'application/json', 'Host': 'switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com', 'content-type': 'application/octet-stream', 'inference-header-content-length': '49'}
is_graph = True
async def _predict(
client: InferenceRESTClient,
url,
input_data,
model_name,
headers=None,
is_graph=False,
*,
retries: int = 5,
retry_delay: int = 5,
) -> Union[InferResponse, Dict]:
transient_status_codes = {502, 503, 504}
logger.info("Sending Header = %s", headers)
logger.info("base url = %s", url)
for attempt in range(retries):
try:
> return await client.infer(
url,
input_data,
model_name=model_name,
headers=headers,
is_graph_endpoint=is_graph,
)
common/utils.py:148:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
base_url = 'https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
data = b'{"decision_picker":"ERROR","key1":{"name":"abc"}}', model_name = None
headers = {'Content-Type': 'application/json', 'Host': 'switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com', 'content-type': 'application/octet-stream', 'inference-header-content-length': '49'}
response_headers = None, is_graph_endpoint = True
timeout =
async def infer(
self,
base_url: Union[httpx.URL, str],
data: Union[InferRequest, dict],
model_name: Optional[str] = None,
headers: Optional[Mapping[str, str]] = None,
response_headers: Dict[str, str] = None,
is_graph_endpoint: bool = False,
timeout: Union[float, None, tuple, httpx.Timeout] = httpx.USE_CLIENT_DEFAULT,
) -> Union[InferResponse, Dict]:
"""
Run asynchronous inference using the supplied data.
:param base_url: Base url of the inference server. E.g. https://example.com:443, https://example.com:443/serving
:param data: Input data as InferRequest object.
:param model_name: (optional) Name of the model as string. If is_graph_endpoint is true this can be omitted.
If is_graph_endpoint is False and model_name is None, this will raise ValueError.
:param headers: (optional) HTTP headers to include when sending request.
:param is_graph_endpoint: (optional) If set to True the base_url will be considered as an inference graph
endpoint and will be used as it is for making the request regardless of the
protocol specified in the RESTConfig. The result will be returned as dict object.
Defaults to False.
:param timeout: (optional) The maximum end-to-end time, in seconds, the request is allowed to take. This will
override the timeout in the RESTConfig. The default value is 60 seconds.
To disable timeout explicitly set it to 'None'.
:return: Inference result as InferResponse object or python dict.
:raises HTTPStatusError for response codes other than 2xx.
:raises UnsupportedProtocol if the specified protocol version is not supported.
"""
if is_graph_endpoint:
url = base_url
elif model_name is None:
raise ValueError("model_name should not be 'None'")
elif is_v1(self._config.protocol):
url = self._construct_url(
base_url, f"{self._config.protocol}/models/{model_name}:predict"
)
elif is_v2(self._config.protocol):
url = self._construct_url(
base_url, f"{self._config.protocol}/models/{model_name}/infer"
)
else:
raise UnsupportedProtocol(self._config.protocol)
if self._config.verbose:
logger.info("url: %s", url)
logger.info("request data: %s", data)
if isinstance(data, InferRequest):
data, json_length = data.to_rest()
if json_length:
headers = headers or {}
headers[INFERENCE_CONTENT_LENGTH_HEADER] = str(json_length)
headers["content-type"] = "application/octet-stream"
if isinstance(data, dict):
data = orjson.dumps(data)
headers = headers or {}
headers.setdefault(INFERENCE_CONTENT_LENGTH_HEADER, str(len(data)))
headers.setdefault("content-type", "application/octet-stream")
response = await self._client.post(
url, content=data, headers=headers, timeout=timeout
)
if self._config.verbose:
logger.info(
"response code: %s, content: %s", response.status_code, response.text
)
if not response.is_success:
> raise self._construct_http_status_error(response)
E httpx.HTTPStatusError: , '503 Service Unavailable' for url 'https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
../../python/kserve/kserve/inference_client.py:511: HTTPStatusError
------------------------------ Captured log setup ------------------------------
INFO kserve:conftest.py:40 Logger configured
------------------------------ Captured log call -------------------------------
INFO kserve.trace:test_inference_graph.py:549 Starting test test_ig_scenario5
INFO kserve.trace:test_inference_graph.py:207 SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
INFO kserve.trace:test_inference_graph.py:208 ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
INFO kserve.trace:test_inference_graph.py:554 success_isvc_name is success-200-isvc-e1a28
INFO kserve.trace:test_inference_graph.py:555 error_isvc_name is error-404-isvc-e1a28
INFO kserve.trace:utils.py:335 CI_USE_ISVC_HOST = 1
INFO kserve.trace:utils.py:336 Host from isvc status URL = switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:utils.py:337 Network layer = istio
INFO kserve.trace:utils.py:340 Using external route host: switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:utils.py:144 Sending Header = {'Host': 'switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com', 'Content-Type': 'application/json'}
INFO kserve.trace:utils.py:145 base url = https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:490 url: https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
INFO httpx:_client.py:1786 HTTP Request: POST https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com "HTTP/1.0 503 Service Unavailable"
INFO kserve.trace:inference_client.py:507 response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:166 Transient error on attempt 1/5, retrying in 5s...
INFO kserve.trace:inference_client.py:490 url: https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
INFO httpx:_client.py:1786 HTTP Request: POST https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com "HTTP/1.0 503 Service Unavailable"
INFO kserve.trace:inference_client.py:507 response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:166 Transient error on attempt 2/5, retrying in 5s...
INFO kserve.trace:inference_client.py:490 url: https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
INFO httpx:_client.py:1786 HTTP Request: POST https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com "HTTP/1.0 503 Service Unavailable"
INFO kserve.trace:inference_client.py:507 response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:166 Transient error on attempt 3/5, retrying in 5s...
INFO kserve.trace:inference_client.py:490 url: https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
INFO httpx:_client.py:1786 HTTP Request: POST https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com "HTTP/1.0 503 Service Unavailable"
INFO kserve.trace:inference_client.py:507 response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:166 Transient error on attempt 4/5, retrying in 5s...
INFO kserve.trace:inference_client.py:490 url: https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
INFO httpx:_client.py:1786 HTTP Request: POST https://switch-graph-e1a28-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com "HTTP/1.0 503 Service Unavailable"
INFO kserve.trace:inference_client.py:507 response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
_____________________________ test_inference_graph _____________________________
[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python
rest_v1_client =
@pytest.mark.graph
@pytest.mark.kourier
@pytest.mark.asyncio(scope="session")
async def test_inference_graph(rest_v1_client):
logger.info("Starting test test_inference_graph")
sklearn_name_1 = "isvc-sklearn-graph-1"
sklearn_name_2 = "isvc-sklearn-graph-2"
xgb_name = "isvc-xgboost-graph"
graph_name = "model-chainer"
sklearn_predictor_1 = V1beta1PredictorSpec(
min_replicas=1,
sklearn=V1beta1SKLearnSpec(
storage_uri="gs://kfserving-examples/models/sklearn/1.0/model",
resources=V1ResourceRequirements(
requests={"cpu": "50m", "memory": "128Mi"},
limits={"cpu": "100m", "memory": "256Mi"},
),
args=["--model_name=sklearn"],
),
)
sklearn_predictor_2 = V1beta1PredictorSpec(
min_replicas=1,
sklearn=V1beta1SKLearnSpec(
storage_uri="gs://kfserving-examples/models/sklearn/1.0/model",
resources=V1ResourceRequirements(
requests={"cpu": "50m", "memory": "128Mi"},
limits={"cpu": "100m", "memory": "256Mi"},
),
args=["--model_name", "iris"],
),
)
sklearn_isvc_1 = V1beta1InferenceService(
api_version=constants.KSERVE_V1BETA1,
kind=constants.KSERVE_KIND_INFERENCESERVICE,
metadata=client.V1ObjectMeta(
name=sklearn_name_1, namespace=KSERVE_TEST_NAMESPACE
),
spec=V1beta1InferenceServiceSpec(predictor=sklearn_predictor_1),
)
sklearn_isvc_2 = V1beta1InferenceService(
api_version=constants.KSERVE_V1BETA1,
kind=constants.KSERVE_KIND_INFERENCESERVICE,
metadata=client.V1ObjectMeta(
name=sklearn_name_2, namespace=KSERVE_TEST_NAMESPACE
),
spec=V1beta1InferenceServiceSpec(predictor=sklearn_predictor_2),
)
xgb_predictor = V1beta1PredictorSpec(
min_replicas=1,
xgboost=V1beta1XGBoostSpec(
storage_uri="gs://kfserving-examples/models/xgboost/1.5/model",
resources=V1ResourceRequirements(
requests={"cpu": "50m", "memory": "128Mi"},
limits={"cpu": "100m", "memory": "256Mi"},
),
),
)
xgb_isvc = V1beta1InferenceService(
api_version=constants.KSERVE_V1BETA1,
kind=constants.KSERVE_KIND_INFERENCESERVICE,
metadata=client.V1ObjectMeta(name=xgb_name, namespace=KSERVE_TEST_NAMESPACE),
spec=V1beta1InferenceServiceSpec(predictor=xgb_predictor),
)
nodes = {
"root": V1alpha1InferenceRouter(
router_type="Sequence",
steps=[
V1alpha1InferenceStep(
service_name=sklearn_name_1,
dependency="Hard",
),
V1alpha1InferenceStep(
service_name=xgb_name,
data="$request",
dependency="Hard",
),
V1alpha1InferenceStep(
service_name=sklearn_name_2,
data="$request",
dependency="Hard",
),
],
)
}
graph_spec = V1alpha1InferenceGraphSpec(
nodes=nodes,
)
ig = V1alpha1InferenceGraph(
api_version=constants.KSERVE_V1ALPHA1,
kind=constants.KSERVE_KIND_INFERENCEGRAPH,
metadata=client.V1ObjectMeta(name=graph_name, namespace=KSERVE_TEST_NAMESPACE),
spec=graph_spec,
)
kserve_client = KServeClient(
config_file=os.environ.get("KUBECONFIG", "~/.kube/config")
)
kserve_client.create(sklearn_isvc_1)
kserve_client.create(xgb_isvc)
kserve_client.create(sklearn_isvc_2)
kserve_client.wait_isvc_ready(sklearn_name_1, namespace=KSERVE_TEST_NAMESPACE)
kserve_client.wait_isvc_ready(xgb_name, namespace=KSERVE_TEST_NAMESPACE)
kserve_client.wait_isvc_ready(sklearn_name_2, namespace=KSERVE_TEST_NAMESPACE)
kserve_client.create_inference_graph(ig)
kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE)
> res = await predict_ig(
rest_v1_client,
graph_name,
os.path.join(IG_TEST_RESOURCES_BASE_LOCATION, "iris_input.json"),
)
graph/test_inference_graph.py:149:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
client =
ig_name = 'model-chainer', input_path = 'graph/test-resources/iris_input.json'
version = 'v1alpha1', network_layer = 'istio'
async def predict_ig(
client: InferenceRESTClient,
ig_name,
input_path,
version=constants.KSERVE_V1ALPHA1_VERSION,
network_layer: str = "istio",
) -> Union[InferResponse, Dict]:
kserve_client = KServeClient(
config_file=os.environ.get("KUBECONFIG", "~/.kube/config")
)
ig = kserve_client.get_inference_graph(
ig_name,
namespace=KSERVE_TEST_NAMESPACE,
version=version,
)
scheme, cluster_ip, host, _ = get_isvc_endpoint(ig, network_layer)
url = f"{scheme}://{cluster_ip}"
> return await predict(client, url, host, input_path, is_graph=True)
common/utils.py:193:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
client =
url = 'https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
host = 'model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
input = 'graph/test-resources/iris_input.json', model_name = None
is_batch = False, is_graph = True
async def predict(
client: InferenceRESTClient,
url,
host,
input: Union[str, InferRequest],
model_name=None,
is_batch=False,
is_graph=False,
) -> Union[InferResponse, Dict, List[Union[Dict, InferResponse]]]:
if isinstance(input, str):
with open(input) as json_file:
data = json.load(json_file)
else:
data = input
headers = {"Host": host, "Content-Type": "application/json"}
if is_batch:
with futures.ThreadPoolExecutor(max_workers=4) as executor:
loop = asyncio.get_running_loop()
future_list = [
await loop.run_in_executor(
executor,
_predict,
client,
url,
input_data,
model_name,
headers,
is_graph,
)
for input_data in data
]
result = await asyncio.gather(*future_list)
else:
> result = await _predict(
client,
url,
data,
model_name=model_name,
headers=headers,
is_graph=is_graph,
)
common/utils.py:120:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
client =
url = 'https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
input_data = {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
model_name = None
headers = {'Content-Type': 'application/json', 'Host': 'model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com', 'content-type': 'application/octet-stream', 'inference-header-content-length': '51'}
is_graph = True
async def _predict(
client: InferenceRESTClient,
url,
input_data,
model_name,
headers=None,
is_graph=False,
*,
retries: int = 5,
retry_delay: int = 5,
) -> Union[InferResponse, Dict]:
transient_status_codes = {502, 503, 504}
logger.info("Sending Header = %s", headers)
logger.info("base url = %s", url)
for attempt in range(retries):
try:
> return await client.infer(
url,
input_data,
model_name=model_name,
headers=headers,
is_graph_endpoint=is_graph,
)
common/utils.py:148:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
base_url = 'https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
data = b'{"instances":[[6.8,2.8,4.8,1.4],[6.0,3.4,4.5,1.6]]}', model_name = None
headers = {'Content-Type': 'application/json', 'Host': 'model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com', 'content-type': 'application/octet-stream', 'inference-header-content-length': '51'}
response_headers = None, is_graph_endpoint = True
timeout =
async def infer(
self,
base_url: Union[httpx.URL, str],
data: Union[InferRequest, dict],
model_name: Optional[str] = None,
headers: Optional[Mapping[str, str]] = None,
response_headers: Dict[str, str] = None,
is_graph_endpoint: bool = False,
timeout: Union[float, None, tuple, httpx.Timeout] = httpx.USE_CLIENT_DEFAULT,
) -> Union[InferResponse, Dict]:
"""
Run asynchronous inference using the supplied data.
:param base_url: Base url of the inference server. E.g. https://example.com:443, https://example.com:443/serving
:param data: Input data as InferRequest object.
:param model_name: (optional) Name of the model as string. If is_graph_endpoint is true this can be omitted.
If is_graph_endpoint is False and model_name is None, this will raise ValueError.
:param headers: (optional) HTTP headers to include when sending request.
:param is_graph_endpoint: (optional) If set to True the base_url will be considered as an inference graph
endpoint and will be used as it is for making the request regardless of the
protocol specified in the RESTConfig. The result will be returned as dict object.
Defaults to False.
:param timeout: (optional) The maximum end-to-end time, in seconds, the request is allowed to take. This will
override the timeout in the RESTConfig. The default value is 60 seconds.
To disable timeout explicitly set it to 'None'.
:return: Inference result as InferResponse object or python dict.
:raises HTTPStatusError for response codes other than 2xx.
:raises UnsupportedProtocol if the specified protocol version is not supported.
"""
if is_graph_endpoint:
url = base_url
elif model_name is None:
raise ValueError("model_name should not be 'None'")
elif is_v1(self._config.protocol):
url = self._construct_url(
base_url, f"{self._config.protocol}/models/{model_name}:predict"
)
elif is_v2(self._config.protocol):
url = self._construct_url(
base_url, f"{self._config.protocol}/models/{model_name}/infer"
)
else:
raise UnsupportedProtocol(self._config.protocol)
if self._config.verbose:
logger.info("url: %s", url)
logger.info("request data: %s", data)
if isinstance(data, InferRequest):
data, json_length = data.to_rest()
if json_length:
headers = headers or {}
headers[INFERENCE_CONTENT_LENGTH_HEADER] = str(json_length)
headers["content-type"] = "application/octet-stream"
if isinstance(data, dict):
data = orjson.dumps(data)
headers = headers or {}
headers.setdefault(INFERENCE_CONTENT_LENGTH_HEADER, str(len(data)))
headers.setdefault("content-type", "application/octet-stream")
response = await self._client.post(
url, content=data, headers=headers, timeout=timeout
)
if self._config.verbose:
logger.info(
"response code: %s, content: %s", response.status_code, response.text
)
if not response.is_success:
> raise self._construct_http_status_error(response)
E httpx.HTTPStatusError: , '503 Service Unavailable' for url 'https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
../../python/kserve/kserve/inference_client.py:511: HTTPStatusError
------------------------------ Captured log setup ------------------------------
INFO kserve:conftest.py:40 Logger configured
------------------------------ Captured log call -------------------------------
INFO kserve.trace:test_inference_graph.py:43 Starting test test_inference_graph
INFO kserve.trace:utils.py:335 CI_USE_ISVC_HOST = 1
INFO kserve.trace:utils.py:336 Host from isvc status URL = model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:utils.py:337 Network layer = istio
INFO kserve.trace:utils.py:340 Using external route host: model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:utils.py:144 Sending Header = {'Host': 'model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com', 'Content-Type': 'application/json'}
INFO kserve.trace:utils.py:145 base url = https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:490 url: https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
INFO httpx:_client.py:1786 HTTP Request: POST https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com "HTTP/1.0 503 Service Unavailable"
INFO kserve.trace:inference_client.py:507 response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:166 Transient error on attempt 1/5, retrying in 5s...
INFO kserve.trace:inference_client.py:490 url: https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
INFO httpx:_client.py:1786 HTTP Request: POST https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com "HTTP/1.0 503 Service Unavailable"
INFO kserve.trace:inference_client.py:507 response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:166 Transient error on attempt 2/5, retrying in 5s...
INFO kserve.trace:inference_client.py:490 url: https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
INFO httpx:_client.py:1786 HTTP Request: POST https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com "HTTP/1.0 503 Service Unavailable"
INFO kserve.trace:inference_client.py:507 response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:166 Transient error on attempt 3/5, retrying in 5s...
INFO kserve.trace:inference_client.py:490 url: https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
INFO httpx:_client.py:1786 HTTP Request: POST https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com "HTTP/1.0 503 Service Unavailable"
INFO kserve.trace:inference_client.py:507 response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:166 Transient error on attempt 4/5, retrying in 5s...
INFO kserve.trace:inference_client.py:490 url: https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
INFO httpx:_client.py:1786 HTTP Request: POST https://model-chainer-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com "HTTP/1.0 503 Service Unavailable"
INFO kserve.trace:inference_client.py:507 response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
______________________________ test_ig_scenario6 _______________________________
[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python
rest_v1_client =
@pytest.mark.graph
@pytest.mark.kourier
@pytest.mark.asyncio(scope="session")
async def test_ig_scenario6(rest_v1_client):
"""
Scenario: Switch graph where a match would happen for error node and then error would return and IG will NOT
continue execution and call the next step in the flow as error step will be a HARD dependency.
Expectation: IG will return response of success_isvc.
"""
logger.info("Starting test test_ig_scenario6")
suffix = str(uuid.uuid4())[1:6]
success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test(
suffix
)
logger.info(f"success_isvc_name is {success_isvc_name}")
logger.info(f"error_isvc_name is {error_isvc_name}")
kserve_client = KServeClient(
config_file=os.environ.get("KUBECONFIG", "~/.kube/config")
)
kserve_client.create(success_isvc)
kserve_client.create(error_isvc)
# Create graph
graph_name = "-".join(["switch-graph", suffix])
# Because we run from test/e2e location in run-e2e-tests.sh
deployment_yaml_path = os.path.join(
IG_TEST_RESOURCES_BASE_LOCATION, "ig_test_switch_scenario_6.yaml"
)
# Read YAML file
with open(deployment_yaml_path, "r") as stream:
file_content = stream.read()
resource_template = Template(file_content)
substitutions = {
"graph_name": graph_name,
"error_404_isvc_id": error_isvc_name,
"success_200_isvc_id": success_isvc_name,
}
resource_body_after_rendering = yaml.safe_load(
resource_template.render(substitutions)
)
kserve_client.wait_isvc_ready(success_isvc_name, namespace=KSERVE_TEST_NAMESPACE)
kserve_client.wait_isvc_ready(error_isvc_name, namespace=KSERVE_TEST_NAMESPACE)
create_ig_using_custom_object_api(resource_body_after_rendering)
kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE)
with pytest.raises(HTTPStatusError) as exc_info:
await predict_ig(
rest_v1_client,
graph_name,
os.path.join(
IG_TEST_RESOURCES_BASE_LOCATION, "switch_call_error_picker_input.json"
),
)
> assert exc_info.value.response.json() == {"detail": "Intentional 404 code"}
graph/test_inference_graph.py:663:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = , kwargs = {}
def json(self, **kwargs: typing.Any) -> typing.Any:
> return jsonlib.loads(self.content, **kwargs)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx/_models.py:766:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
s = '\r\n \r\n \r\n\r\n
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:166 Transient error on attempt 1/5, retrying in 5s...
INFO kserve.trace:inference_client.py:490 url: https://switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
INFO httpx:_client.py:1786 HTTP Request: POST https://switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com "HTTP/1.0 503 Service Unavailable"
INFO kserve.trace:inference_client.py:507 response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:166 Transient error on attempt 2/5, retrying in 5s...
INFO kserve.trace:inference_client.py:490 url: https://switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
INFO httpx:_client.py:1786 HTTP Request: POST https://switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com "HTTP/1.0 503 Service Unavailable"
INFO kserve.trace:inference_client.py:507 response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:166 Transient error on attempt 3/5, retrying in 5s...
INFO kserve.trace:inference_client.py:490 url: https://switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
INFO httpx:_client.py:1786 HTTP Request: POST https://switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com "HTTP/1.0 503 Service Unavailable"
INFO kserve.trace:inference_client.py:507 response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:166 Transient error on attempt 4/5, retrying in 5s...
INFO kserve.trace:inference_client.py:490 url: https://switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'decision_picker': 'ERROR', 'key1': {'name': 'abc'}}
INFO httpx:_client.py:1786 HTTP Request: POST https://switch-graph-4f739-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com "HTTP/1.0 503 Service Unavailable"
INFO kserve.trace:inference_client.py:507 response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
______________________________ test_ig_scenario1 _______________________________
[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python
rest_v1_client =
@pytest.mark.graph
@pytest.mark.kourier
@pytest.mark.asyncio(scope="session")
async def test_ig_scenario1(rest_v1_client):
"""
Scenario: Sequence graph with 2 steps that are both soft dependencies.
success_isvc(soft) -> error_isvc (soft)
We are not marking steps as soft or hard explicitly so this will test that default behavior of steps being soft
is as expected.
Expectation: IG will return response of error_isvc and predict_ig will raise exception
:return:
"""
logger.info("Starting test test_ig_scenario1")
suffix = str(uuid.uuid4())[1:6]
success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test(
suffix
)
logger.info(f"success_isvc_name is {success_isvc_name}")
logger.info(f"error_isvc_name is {error_isvc_name}")
# Create graph
graph_name = "-".join(["sequence-graph", suffix])
nodes = {
"root": V1alpha1InferenceRouter(
router_type="Sequence",
steps=[
V1alpha1InferenceStep(
service_name=success_isvc_name,
),
V1alpha1InferenceStep(
service_name=error_isvc_name,
data="$request",
),
],
)
}
graph_spec = V1alpha1InferenceGraphSpec(
nodes=nodes,
)
ig = V1alpha1InferenceGraph(
api_version=constants.KSERVE_V1ALPHA1,
kind=constants.KSERVE_KIND_INFERENCEGRAPH,
metadata=client.V1ObjectMeta(name=graph_name, namespace=KSERVE_TEST_NAMESPACE),
spec=graph_spec,
)
kserve_client = KServeClient(
config_file=os.environ.get("KUBECONFIG", "~/.kube/config")
)
kserve_client.create(success_isvc)
kserve_client.create(error_isvc)
kserve_client.wait_isvc_ready(success_isvc_name, namespace=KSERVE_TEST_NAMESPACE)
kserve_client.wait_isvc_ready(error_isvc_name, namespace=KSERVE_TEST_NAMESPACE)
kserve_client.create_inference_graph(ig)
kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE)
with pytest.raises(HTTPStatusError) as exc_info:
await predict_ig(
rest_v1_client,
graph_name,
os.path.join(
IG_TEST_RESOURCES_BASE_LOCATION, "custom_predictor_input.json"
),
)
> assert exc_info.value.response.json() == {"detail": "Intentional 404 code"}
graph/test_inference_graph.py:294:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = , kwargs = {}
def json(self, **kwargs: typing.Any) -> typing.Any:
> return jsonlib.loads(self.content, **kwargs)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx/_models.py:766:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
s = '\r\n \r\n \r\n\r\n
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:166 Transient error on attempt 1/5, retrying in 5s...
INFO kserve.trace:inference_client.py:490 url: https://sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
INFO httpx:_client.py:1786 HTTP Request: POST https://sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com "HTTP/1.0 503 Service Unavailable"
INFO kserve.trace:inference_client.py:507 response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:166 Transient error on attempt 2/5, retrying in 5s...
INFO kserve.trace:inference_client.py:490 url: https://sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
INFO httpx:_client.py:1786 HTTP Request: POST https://sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com "HTTP/1.0 503 Service Unavailable"
INFO kserve.trace:inference_client.py:507 response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:166 Transient error on attempt 3/5, retrying in 5s...
INFO kserve.trace:inference_client.py:490 url: https://sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
INFO httpx:_client.py:1786 HTTP Request: POST https://sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com "HTTP/1.0 503 Service Unavailable"
INFO kserve.trace:inference_client.py:507 response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:166 Transient error on attempt 4/5, retrying in 5s...
INFO kserve.trace:inference_client.py:490 url: https://sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
INFO httpx:_client.py:1786 HTTP Request: POST https://sequence-graph-ed38c-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com "HTTP/1.0 503 Service Unavailable"
INFO kserve.trace:inference_client.py:507 response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
______________________________ test_ig_scenario7 _______________________________
[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python
@contextlib.contextmanager
def map_httpcore_exceptions() -> typing.Iterator[None]:
try:
> yield
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx/_transports/default.py:72:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
request =
async def handle_async_request(
self,
request: Request,
) -> Response:
assert isinstance(request.stream, AsyncByteStream)
req = httpcore.Request(
method=request.method,
url=httpcore.URL(
scheme=request.url.raw_scheme,
host=request.url.raw_host,
port=request.url.port,
target=request.url.raw_path,
),
headers=request.headers.raw,
content=request.stream,
extensions=request.extensions,
)
with map_httpcore_exceptions():
> resp = await self._pool.handle_async_request(req)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx/_transports/default.py:377:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
request =
async def handle_async_request(self, request: Request) -> Response:
"""
Send an HTTP request, and return an HTTP response.
This is the core implementation that is called into by `.request()` or `.stream()`.
"""
scheme = request.url.scheme.decode()
if scheme == "":
raise UnsupportedProtocol(
"Request URL is missing an 'http://' or 'https://' protocol."
)
if scheme not in ("http", "https", "ws", "wss"):
raise UnsupportedProtocol(
f"Request URL has an unsupported protocol '{scheme}://'."
)
timeouts = request.extensions.get("timeout", {})
timeout = timeouts.get("pool", None)
with self._optional_thread_lock:
# Add the incoming request to our request queue.
pool_request = AsyncPoolRequest(request)
self._requests.append(pool_request)
try:
while True:
with self._optional_thread_lock:
# Assign incoming requests to available connections,
# closing or creating new connections as required.
closing = self._assign_requests_to_connections()
await self._close_connections(closing)
# Wait until this request has an assigned connection.
connection = await pool_request.wait_for_connection(timeout=timeout)
try:
# Send the request on the assigned connection.
response = await connection.handle_async_request(
pool_request.request
)
except ConnectionNotAvailable:
# In some cases a connection may initially be available to
# handle a request, but then become unavailable.
#
# In this case we clear the connection and try again.
pool_request.clear_connection()
else:
break # pragma: nocover
except BaseException as exc:
with self._optional_thread_lock:
# For any exception or cancellation we remove the request from
# the queue, and then re-assign requests to connections.
self._requests.remove(pool_request)
closing = self._assign_requests_to_connections()
await self._close_connections(closing)
> raise exc from None
../../python/kserve/.venv/lib64/python3.11/site-packages/httpcore/_async/connection_pool.py:256:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
request =
async def handle_async_request(self, request: Request) -> Response:
"""
Send an HTTP request, and return an HTTP response.
This is the core implementation that is called into by `.request()` or `.stream()`.
"""
scheme = request.url.scheme.decode()
if scheme == "":
raise UnsupportedProtocol(
"Request URL is missing an 'http://' or 'https://' protocol."
)
if scheme not in ("http", "https", "ws", "wss"):
raise UnsupportedProtocol(
f"Request URL has an unsupported protocol '{scheme}://'."
)
timeouts = request.extensions.get("timeout", {})
timeout = timeouts.get("pool", None)
with self._optional_thread_lock:
# Add the incoming request to our request queue.
pool_request = AsyncPoolRequest(request)
self._requests.append(pool_request)
try:
while True:
with self._optional_thread_lock:
# Assign incoming requests to available connections,
# closing or creating new connections as required.
closing = self._assign_requests_to_connections()
await self._close_connections(closing)
# Wait until this request has an assigned connection.
connection = await pool_request.wait_for_connection(timeout=timeout)
try:
# Send the request on the assigned connection.
> response = await connection.handle_async_request(
pool_request.request
)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpcore/_async/connection_pool.py:236:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = , request =
async def handle_async_request(self, request: Request) -> Response:
if not self.can_handle_request(request.url.origin):
raise RuntimeError(
f"Attempted to send request to {request.url.origin} on connection to {self._origin}"
)
try:
async with self._request_lock:
if self._connection is None:
stream = await self._connect(request)
ssl_object = stream.get_extra_info("ssl_object")
http2_negotiated = (
ssl_object is not None
and ssl_object.selected_alpn_protocol() == "h2"
)
if http2_negotiated or (self._http2 and not self._http1):
from .http2 import AsyncHTTP2Connection
self._connection = AsyncHTTP2Connection(
origin=self._origin,
stream=stream,
keepalive_expiry=self._keepalive_expiry,
)
else:
self._connection = AsyncHTTP11Connection(
origin=self._origin,
stream=stream,
keepalive_expiry=self._keepalive_expiry,
)
except BaseException as exc:
self._connect_failed = True
> raise exc
../../python/kserve/.venv/lib64/python3.11/site-packages/httpcore/_async/connection.py:101:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = , request =
async def handle_async_request(self, request: Request) -> Response:
if not self.can_handle_request(request.url.origin):
raise RuntimeError(
f"Attempted to send request to {request.url.origin} on connection to {self._origin}"
)
try:
async with self._request_lock:
if self._connection is None:
> stream = await self._connect(request)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpcore/_async/connection.py:78:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = , request =
async def _connect(self, request: Request) -> AsyncNetworkStream:
timeouts = request.extensions.get("timeout", {})
sni_hostname = request.extensions.get("sni_hostname", None)
timeout = timeouts.get("connect", None)
retries_left = self._retries
delays = exponential_backoff(factor=RETRIES_BACKOFF_FACTOR)
while True:
try:
if self._uds is None:
kwargs = {
"host": self._origin.host.decode("ascii"),
"port": self._origin.port,
"local_address": self._local_address,
"timeout": timeout,
"socket_options": self._socket_options,
}
async with Trace("connect_tcp", logger, request, kwargs) as trace:
> stream = await self._network_backend.connect_tcp(**kwargs)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpcore/_async/connection.py:124:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
host = 'ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
port = 443, timeout = 180, local_address = None, socket_options = None
async def connect_tcp(
self,
host: str,
port: int,
timeout: float | None = None,
local_address: str | None = None,
socket_options: typing.Iterable[SOCKET_OPTION] | None = None,
) -> AsyncNetworkStream:
await self._init_backend()
> return await self._backend.connect_tcp(
host,
port,
timeout=timeout,
local_address=local_address,
socket_options=socket_options,
)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpcore/_backends/auto.py:31:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
host = 'ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
port = 443, timeout = 180, local_address = None, socket_options = []
async def connect_tcp(
self,
host: str,
port: int,
timeout: float | None = None,
local_address: str | None = None,
socket_options: typing.Iterable[SOCKET_OPTION] | None = None,
) -> AsyncNetworkStream: # pragma: nocover
if socket_options is None:
socket_options = []
exc_map = {
TimeoutError: ConnectTimeout,
OSError: ConnectError,
anyio.BrokenResourceError: ConnectError,
}
> with map_exceptions(exc_map):
../../python/kserve/.venv/lib64/python3.11/site-packages/httpcore/_backends/anyio.py:113:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
typ =
value = gaierror(-2, 'Name or service not known')
traceback =
def __exit__(self, typ, value, traceback):
if typ is None:
try:
next(self.gen)
except StopIteration:
return False
else:
try:
raise RuntimeError("generator didn't stop")
finally:
self.gen.close()
else:
if value is None:
# Need to force instantiation so we can reliably
# tell if we get the same exception back
value = typ()
try:
> self.gen.throw(typ, value, traceback)
/usr/lib64/python3.11/contextlib.py:158:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
map = {: , : , : }
@contextlib.contextmanager
def map_exceptions(map: ExceptionMapping) -> typing.Iterator[None]:
try:
yield
except Exception as exc: # noqa: PIE786
for from_exc, to_exc in map.items():
if isinstance(exc, from_exc):
> raise to_exc(exc) from exc
E httpcore.ConnectError: [Errno -2] Name or service not known
../../python/kserve/.venv/lib64/python3.11/site-packages/httpcore/_exceptions.py:14: ConnectError
The above exception was the direct cause of the following exception:
rest_v1_client =
@pytest.mark.graph
@pytest.mark.kourier
@pytest.mark.asyncio(scope="session")
async def test_ig_scenario7(rest_v1_client):
"""
Scenario: Ensemble graph with 2 steps, where both the steps are soft deps.
Expectation: IG will return combined response of both the steps.
"""
logger.info("Starting test test_ig_scenario7")
suffix = str(uuid.uuid4())[1:6]
success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test(
suffix
)
logger.info(f"success_isvc_name is {success_isvc_name}")
logger.info(f"error_isvc_name is {error_isvc_name}")
kserve_client = KServeClient(
config_file=os.environ.get("KUBECONFIG", "~/.kube/config")
)
kserve_client.create(success_isvc)
kserve_client.create(error_isvc)
# Create graph
graph_name = "-".join(["ensemble-graph", suffix])
# Because we run from test/e2e location in run-e2e-tests.sh
deployment_yaml_path = os.path.join(
IG_TEST_RESOURCES_BASE_LOCATION, "ig_test_ensemble_scenario_7.yaml"
)
# Read YAML file
with open(deployment_yaml_path, "r") as stream:
file_content = stream.read()
resource_template = Template(file_content)
substitutions = {
"graph_name": graph_name,
"error_404_isvc_id": error_isvc_name,
"success_200_isvc_id": success_isvc_name,
}
resource_body_after_rendering = yaml.safe_load(
resource_template.render(substitutions)
)
kserve_client.wait_isvc_ready(success_isvc_name, namespace=KSERVE_TEST_NAMESPACE)
kserve_client.wait_isvc_ready(error_isvc_name, namespace=KSERVE_TEST_NAMESPACE)
create_ig_using_custom_object_api(resource_body_after_rendering)
kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE)
> response = await predict_ig(
rest_v1_client,
graph_name,
os.path.join(
IG_TEST_RESOURCES_BASE_LOCATION, "switch_call_success_picker_input.json"
),
)
graph/test_inference_graph.py:721:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
client =
ig_name = 'ensemble-graph-42610'
input_path = 'graph/test-resources/switch_call_success_picker_input.json'
version = 'v1alpha1', network_layer = 'istio'
async def predict_ig(
client: InferenceRESTClient,
ig_name,
input_path,
version=constants.KSERVE_V1ALPHA1_VERSION,
network_layer: str = "istio",
) -> Union[InferResponse, Dict]:
kserve_client = KServeClient(
config_file=os.environ.get("KUBECONFIG", "~/.kube/config")
)
ig = kserve_client.get_inference_graph(
ig_name,
namespace=KSERVE_TEST_NAMESPACE,
version=version,
)
scheme, cluster_ip, host, _ = get_isvc_endpoint(ig, network_layer)
url = f"{scheme}://{cluster_ip}"
> return await predict(client, url, host, input_path, is_graph=True)
common/utils.py:193:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
client =
url = 'https://ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
host = 'ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
input = 'graph/test-resources/switch_call_success_picker_input.json'
model_name = None, is_batch = False, is_graph = True
async def predict(
client: InferenceRESTClient,
url,
host,
input: Union[str, InferRequest],
model_name=None,
is_batch=False,
is_graph=False,
) -> Union[InferResponse, Dict, List[Union[Dict, InferResponse]]]:
if isinstance(input, str):
with open(input) as json_file:
data = json.load(json_file)
else:
data = input
headers = {"Host": host, "Content-Type": "application/json"}
if is_batch:
with futures.ThreadPoolExecutor(max_workers=4) as executor:
loop = asyncio.get_running_loop()
future_list = [
await loop.run_in_executor(
executor,
_predict,
client,
url,
input_data,
model_name,
headers,
is_graph,
)
for input_data in data
]
result = await asyncio.gather(*future_list)
else:
> result = await _predict(
client,
url,
data,
model_name=model_name,
headers=headers,
is_graph=is_graph,
)
common/utils.py:120:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
client =
url = 'https://ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
input_data = {'decision_picker': 'SUCCESS', 'key1': {'name': 'abc'}}
model_name = None
headers = {'Content-Type': 'application/json', 'Host': 'ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd...c922f58bfea.prod.konfluxeaas.com', 'content-type': 'application/octet-stream', 'inference-header-content-length': '51'}
is_graph = True
async def _predict(
client: InferenceRESTClient,
url,
input_data,
model_name,
headers=None,
is_graph=False,
*,
retries: int = 5,
retry_delay: int = 5,
) -> Union[InferResponse, Dict]:
transient_status_codes = {502, 503, 504}
logger.info("Sending Header = %s", headers)
logger.info("base url = %s", url)
for attempt in range(retries):
try:
> return await client.infer(
url,
input_data,
model_name=model_name,
headers=headers,
is_graph_endpoint=is_graph,
)
common/utils.py:148:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
base_url = 'https://ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
data = b'{"decision_picker":"SUCCESS","key1":{"name":"abc"}}', model_name = None
headers = {'Content-Type': 'application/json', 'Host': 'ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd...c922f58bfea.prod.konfluxeaas.com', 'content-type': 'application/octet-stream', 'inference-header-content-length': '51'}
response_headers = None, is_graph_endpoint = True
timeout =
async def infer(
self,
base_url: Union[httpx.URL, str],
data: Union[InferRequest, dict],
model_name: Optional[str] = None,
headers: Optional[Mapping[str, str]] = None,
response_headers: Dict[str, str] = None,
is_graph_endpoint: bool = False,
timeout: Union[float, None, tuple, httpx.Timeout] = httpx.USE_CLIENT_DEFAULT,
) -> Union[InferResponse, Dict]:
"""
Run asynchronous inference using the supplied data.
:param base_url: Base url of the inference server. E.g. https://example.com:443, https://example.com:443/serving
:param data: Input data as InferRequest object.
:param model_name: (optional) Name of the model as string. If is_graph_endpoint is true this can be omitted.
If is_graph_endpoint is False and model_name is None, this will raise ValueError.
:param headers: (optional) HTTP headers to include when sending request.
:param is_graph_endpoint: (optional) If set to True the base_url will be considered as an inference graph
endpoint and will be used as it is for making the request regardless of the
protocol specified in the RESTConfig. The result will be returned as dict object.
Defaults to False.
:param timeout: (optional) The maximum end-to-end time, in seconds, the request is allowed to take. This will
override the timeout in the RESTConfig. The default value is 60 seconds.
To disable timeout explicitly set it to 'None'.
:return: Inference result as InferResponse object or python dict.
:raises HTTPStatusError for response codes other than 2xx.
:raises UnsupportedProtocol if the specified protocol version is not supported.
"""
if is_graph_endpoint:
url = base_url
elif model_name is None:
raise ValueError("model_name should not be 'None'")
elif is_v1(self._config.protocol):
url = self._construct_url(
base_url, f"{self._config.protocol}/models/{model_name}:predict"
)
elif is_v2(self._config.protocol):
url = self._construct_url(
base_url, f"{self._config.protocol}/models/{model_name}/infer"
)
else:
raise UnsupportedProtocol(self._config.protocol)
if self._config.verbose:
logger.info("url: %s", url)
logger.info("request data: %s", data)
if isinstance(data, InferRequest):
data, json_length = data.to_rest()
if json_length:
headers = headers or {}
headers[INFERENCE_CONTENT_LENGTH_HEADER] = str(json_length)
headers["content-type"] = "application/octet-stream"
if isinstance(data, dict):
data = orjson.dumps(data)
headers = headers or {}
headers.setdefault(INFERENCE_CONTENT_LENGTH_HEADER, str(len(data)))
headers.setdefault("content-type", "application/octet-stream")
> response = await self._client.post(
url, content=data, headers=headers, timeout=timeout
)
../../python/kserve/kserve/inference_client.py:503:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
url = 'https://ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
async def post(
self,
url: URL | str,
*,
content: RequestContent | None = None,
data: RequestData | None = None,
files: RequestFiles | None = None,
json: typing.Any | None = None,
params: QueryParamTypes | None = None,
headers: HeaderTypes | None = None,
cookies: CookieTypes | None = None,
auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT,
follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT,
timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT,
extensions: RequestExtensions | None = None,
) -> Response:
"""
Send a `POST` request.
**Parameters**: See `httpx.request`.
"""
> return await self.request(
"POST",
url,
content=content,
data=data,
files=files,
json=json,
params=params,
headers=headers,
cookies=cookies,
auth=auth,
follow_redirects=follow_redirects,
timeout=timeout,
extensions=extensions,
)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx/_client.py:1905:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = , method = 'POST'
url = 'https://ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
async def request(
self,
method: str,
url: URL | str,
*,
content: RequestContent | None = None,
data: RequestData | None = None,
files: RequestFiles | None = None,
json: typing.Any | None = None,
params: QueryParamTypes | None = None,
headers: HeaderTypes | None = None,
cookies: CookieTypes | None = None,
auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT,
follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT,
timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT,
extensions: RequestExtensions | None = None,
) -> Response:
"""
Build and send a request.
Equivalent to:
```python
request = client.build_request(...)
response = await client.send(request, ...)
```
See `AsyncClient.build_request()`, `AsyncClient.send()`
and [Merging of configuration][0] for how the various parameters
are merged with client-level configuration.
[0]: /advanced/clients/#merging-of-configuration
"""
if cookies is not None: # pragma: no cover
message = (
"Setting per-request cookies=<...> is being deprecated, because "
"the expected behaviour on cookie persistence is ambiguous. Set "
"cookies directly on the client instance instead."
)
warnings.warn(message, DeprecationWarning)
request = self.build_request(
method=method,
url=url,
content=content,
data=data,
files=files,
json=json,
params=params,
headers=headers,
cookies=cookies,
timeout=timeout,
extensions=extensions,
)
> return await self.send(request, auth=auth, follow_redirects=follow_redirects)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx/_client.py:1585:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
request =
async def send(
self,
request: Request,
*,
stream: bool = False,
auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT,
follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT,
) -> Response:
"""
Send a request.
The request is sent as-is, unmodified.
Typically you'll want to build one with `AsyncClient.build_request()`
so that any client-level configuration is merged into the request,
but passing an explicit `httpx.Request()` is supported as well.
See also: [Request instances][0]
[0]: /advanced/clients/#request-instances
"""
if self._state == ClientState.CLOSED:
raise RuntimeError("Cannot send a request, as the client has been closed.")
self._state = ClientState.OPENED
follow_redirects = (
self.follow_redirects
if isinstance(follow_redirects, UseClientDefault)
else follow_redirects
)
self._set_timeout(request)
auth = self._build_request_auth(request, auth)
> response = await self._send_handling_auth(
request,
auth=auth,
follow_redirects=follow_redirects,
history=[],
)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx/_client.py:1674:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
request =
auth = , follow_redirects = False
history = []
async def _send_handling_auth(
self,
request: Request,
auth: Auth,
follow_redirects: bool,
history: list[Response],
) -> Response:
auth_flow = auth.async_auth_flow(request)
try:
request = await auth_flow.__anext__()
while True:
> response = await self._send_handling_redirects(
request,
follow_redirects=follow_redirects,
history=history,
)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx/_client.py:1702:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
request =
follow_redirects = False, history = []
async def _send_handling_redirects(
self,
request: Request,
follow_redirects: bool,
history: list[Response],
) -> Response:
while True:
if len(history) > self.max_redirects:
raise TooManyRedirects(
"Exceeded maximum allowed redirects.", request=request
)
for hook in self._event_hooks["request"]:
await hook(request)
> response = await self._send_single_request(request)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx/_client.py:1739:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
request =
async def _send_single_request(self, request: Request) -> Response:
"""
Sends a single request, without handling any redirections.
"""
transport = self._transport_for_url(request.url)
timer = Timer()
await timer.async_start()
if not isinstance(request.stream, AsyncByteStream):
raise RuntimeError(
"Attempted to send an sync request with an AsyncClient instance."
)
with request_context(request=request):
> response = await transport.handle_async_request(request)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx/_client.py:1776:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
request =
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
"""Sends an HTTP request, possibly with retries.
Args:
request: The request to perform.
Returns:
The final response.
"""
if self._async_transport is None:
raise RuntimeError("Async request received but no async transport available")
logger.debug("handle_async_request started request=%s", request)
if self.retry.is_retryable_method(request.method):
send_method = partial(self._async_transport.handle_async_request)
> response = await self._retry_operation_async(request, send_method)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx_retries/transport.py:116:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
request =
send_method = functools.partial(>)
async def _retry_operation_async(
self,
request: httpx.Request,
send_method: Callable[..., Coroutine[Any, Any, httpx.Response]],
) -> httpx.Response:
retry = self.retry
response: Union[httpx.Response, Exception, None] = None
while True:
if response is not None:
if isinstance(response, httpx.Response):
await response.aclose()
logger.debug(
"_retry_operation_async retrying request=%s response=%s retry=%s", request, response, retry
)
retry = retry.increment()
await retry.asleep(response)
try:
> response = await send_method(request)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx_retries/transport.py:171:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
request =
async def handle_async_request(
self,
request: Request,
) -> Response:
assert isinstance(request.stream, AsyncByteStream)
req = httpcore.Request(
method=request.method,
url=httpcore.URL(
scheme=request.url.raw_scheme,
host=request.url.raw_host,
port=request.url.port,
target=request.url.raw_path,
),
headers=request.headers.raw,
content=request.stream,
extensions=request.extensions,
)
> with map_httpcore_exceptions():
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx/_transports/default.py:376:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
typ =
value = ConnectError(gaierror(-2, 'Name or service not known'))
traceback =
def __exit__(self, typ, value, traceback):
if typ is None:
try:
next(self.gen)
except StopIteration:
return False
else:
try:
raise RuntimeError("generator didn't stop")
finally:
self.gen.close()
else:
if value is None:
# Need to force instantiation so we can reliably
# tell if we get the same exception back
value = typ()
try:
> self.gen.throw(typ, value, traceback)
/usr/lib64/python3.11/contextlib.py:158:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
@contextlib.contextmanager
def map_httpcore_exceptions() -> typing.Iterator[None]:
try:
yield
except Exception as exc:
mapped_exc = None
for from_exc, to_exc in HTTPCORE_EXC_MAP.items():
if not isinstance(exc, from_exc):
continue
# We want to map to the most specific exception we can find.
# Eg if `exc` is an `httpcore.ReadTimeout`, we want to map to
# `httpx.ReadTimeout`, not just `httpx.TimeoutException`.
if mapped_exc is None or issubclass(to_exc, mapped_exc):
mapped_exc = to_exc
if mapped_exc is None: # pragma: no cover
raise
message = str(exc)
> raise mapped_exc(message) from exc
E httpx.ConnectError: [Errno -2] Name or service not known
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx/_transports/default.py:89: ConnectError
------------------------------ Captured log call -------------------------------
INFO kserve.trace:test_inference_graph.py:680 Starting test test_ig_scenario7
INFO kserve.trace:test_inference_graph.py:207 SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
INFO kserve.trace:test_inference_graph.py:208 ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
INFO kserve.trace:test_inference_graph.py:685 success_isvc_name is success-200-isvc-42610
INFO kserve.trace:test_inference_graph.py:686 error_isvc_name is error-404-isvc-42610
INFO kserve.trace:utils.py:335 CI_USE_ISVC_HOST = 1
INFO kserve.trace:utils.py:336 Host from isvc status URL = ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:utils.py:337 Network layer = istio
INFO kserve.trace:utils.py:340 Using external route host: ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:utils.py:144 Sending Header = {'Host': 'ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com', 'Content-Type': 'application/json'}
INFO kserve.trace:utils.py:145 base url = https://ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:490 url: https://ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'decision_picker': 'SUCCESS', 'key1': {'name': 'abc'}}
INFO httpx:_client.py:1786 HTTP Request: POST https://ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com "HTTP/1.0 503 Service Unavailable"
INFO kserve.trace:inference_client.py:507 response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:166 Transient error on attempt 1/5, retrying in 5s...
INFO kserve.trace:inference_client.py:490 url: https://ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'decision_picker': 'SUCCESS', 'key1': {'name': 'abc'}}
INFO httpx:_client.py:1786 HTTP Request: POST https://ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com "HTTP/1.0 503 Service Unavailable"
INFO kserve.trace:inference_client.py:507 response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:166 Transient error on attempt 2/5, retrying in 5s...
INFO kserve.trace:inference_client.py:490 url: https://ensemble-graph-42610-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'decision_picker': 'SUCCESS', 'key1': {'name': 'abc'}}
______________________________ test_ig_scenario8 _______________________________
[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python
self =
inferenceservice = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
namespace = 'kserve-ci-e2e-test', watch = False, timeout_seconds = 600
def create(
self, inferenceservice, namespace=None, watch=False, timeout_seconds=600
): # pylint:disable=inconsistent-return-statements
"""
Create the inference service
:param inferenceservice: inference service object
:param namespace: defaults to current or default namespace
:param watch: True to watch the created service until timeout elapsed or status is ready
:param timeout_seconds: timeout seconds for watch, default to 600s
:return: created inference service
"""
version = inferenceservice.api_version.split("/")[1]
if namespace is None:
namespace = utils.get_isvc_namespace(inferenceservice)
try:
> outputs = self.api_instance.create_namespaced_custom_object(
constants.KSERVE_GROUP,
version,
namespace,
constants.KSERVE_PLURAL_INFERENCESERVICE,
inferenceservice,
)
../../python/kserve/kserve/api/kserve_client.py:145:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
group = 'serving.kserve.io', version = 'v1beta1'
namespace = 'kserve-ci-e2e-test', plural = 'inferenceservices'
body = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
kwargs = {'_return_http_data_only': True}
def create_namespaced_custom_object(self, group, version, namespace, plural, body, **kwargs): # noqa: E501
"""create_namespaced_custom_object # noqa: E501
Creates a namespace scoped Custom object # noqa: E501
This method makes a synchronous HTTP request by default. To make an
asynchronous HTTP request, please pass async_req=True
>>> thread = api.create_namespaced_custom_object(group, version, namespace, plural, body, async_req=True)
>>> result = thread.get()
:param async_req bool: execute request asynchronously
:param str group: The custom resource's group name (required)
:param str version: The custom resource's version (required)
:param str namespace: The custom resource's namespace (required)
:param str plural: The custom resource's plural name. For TPRs this would be lowercase plural kind. (required)
:param object body: The JSON schema of the Resource to create. (required)
:param str pretty: If 'true', then the output is pretty printed.
:param str dry_run: When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed
:param str field_manager: fieldManager is a name associated with the actor or entity that is making these changes. The value must be less than or 128 characters long, and only contain printable characters, as defined by https://golang.org/pkg/unicode/#IsPrint.
:param str field_validation: fieldValidation instructs the server on how to handle objects in the request (POST/PUT/PATCH) containing unknown or duplicate fields. Valid values are: - Ignore: This will ignore any unknown fields that are silently dropped from the object, and will ignore all but the last duplicate field that the decoder encounters. This is the default behavior prior to v1.23. - Warn: This will send a warning via the standard warning response header for each unknown field that is dropped from the object, and for each duplicate field that is encountered. The request will still succeed if there are no other errors, and will only persist the last of any duplicate fields. This is the default in v1.23+ - Strict: This will fail the request with a BadRequest error if any unknown fields would be dropped from the object, or if any duplicate fields are present. The error returned from the server will contain all unknown and duplicate fields encountered. (optional)
:param _preload_content: if False, the urllib3.HTTPResponse object will
be returned without reading/decoding response
data. Default is True.
:param _request_timeout: timeout setting for this request. If one
number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
:return: object
If the method is called asynchronously,
returns the request thread.
"""
kwargs['_return_http_data_only'] = True
> return self.create_namespaced_custom_object_with_http_info(group, version, namespace, plural, body, **kwargs) # noqa: E501
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/custom_objects_api.py:231:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
group = 'serving.kserve.io', version = 'v1beta1'
namespace = 'kserve-ci-e2e-test', plural = 'inferenceservices'
body = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
kwargs = {'_return_http_data_only': True}
local_var_params = {'_return_http_data_only': True, 'all_params': ['group', 'version', 'namespace', 'plural', 'body', 'pretty', ...], 'au... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}, ...}
all_params = ['group', 'version', 'namespace', 'plural', 'body', 'pretty', ...]
key = '_return_http_data_only', val = True, collection_formats = {}
path_params = {'group': 'serving.kserve.io', 'namespace': 'kserve-ci-e2e-test', 'plural': 'inferenceservices', 'version': 'v1beta1'}
query_params = []
def create_namespaced_custom_object_with_http_info(self, group, version, namespace, plural, body, **kwargs): # noqa: E501
"""create_namespaced_custom_object # noqa: E501
Creates a namespace scoped Custom object # noqa: E501
This method makes a synchronous HTTP request by default. To make an
asynchronous HTTP request, please pass async_req=True
>>> thread = api.create_namespaced_custom_object_with_http_info(group, version, namespace, plural, body, async_req=True)
>>> result = thread.get()
:param async_req bool: execute request asynchronously
:param str group: The custom resource's group name (required)
:param str version: The custom resource's version (required)
:param str namespace: The custom resource's namespace (required)
:param str plural: The custom resource's plural name. For TPRs this would be lowercase plural kind. (required)
:param object body: The JSON schema of the Resource to create. (required)
:param str pretty: If 'true', then the output is pretty printed.
:param str dry_run: When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed
:param str field_manager: fieldManager is a name associated with the actor or entity that is making these changes. The value must be less than or 128 characters long, and only contain printable characters, as defined by https://golang.org/pkg/unicode/#IsPrint.
:param str field_validation: fieldValidation instructs the server on how to handle objects in the request (POST/PUT/PATCH) containing unknown or duplicate fields. Valid values are: - Ignore: This will ignore any unknown fields that are silently dropped from the object, and will ignore all but the last duplicate field that the decoder encounters. This is the default behavior prior to v1.23. - Warn: This will send a warning via the standard warning response header for each unknown field that is dropped from the object, and for each duplicate field that is encountered. The request will still succeed if there are no other errors, and will only persist the last of any duplicate fields. This is the default in v1.23+ - Strict: This will fail the request with a BadRequest error if any unknown fields would be dropped from the object, or if any duplicate fields are present. The error returned from the server will contain all unknown and duplicate fields encountered. (optional)
:param _return_http_data_only: response data without head status code
and headers
:param _preload_content: if False, the urllib3.HTTPResponse object will
be returned without reading/decoding response
data. Default is True.
:param _request_timeout: timeout setting for this request. If one
number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
:return: tuple(object, status_code(int), headers(HTTPHeaderDict))
If the method is called asynchronously,
returns the request thread.
"""
local_var_params = locals()
all_params = [
'group',
'version',
'namespace',
'plural',
'body',
'pretty',
'dry_run',
'field_manager',
'field_validation'
]
all_params.extend(
[
'async_req',
'_return_http_data_only',
'_preload_content',
'_request_timeout'
]
)
for key, val in six.iteritems(local_var_params['kwargs']):
if key not in all_params:
raise ApiTypeError(
"Got an unexpected keyword argument '%s'"
" to method create_namespaced_custom_object" % key
)
local_var_params[key] = val
del local_var_params['kwargs']
# verify the required parameter 'group' is set
if self.api_client.client_side_validation and ('group' not in local_var_params or # noqa: E501
local_var_params['group'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `group` when calling `create_namespaced_custom_object`") # noqa: E501
# verify the required parameter 'version' is set
if self.api_client.client_side_validation and ('version' not in local_var_params or # noqa: E501
local_var_params['version'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `version` when calling `create_namespaced_custom_object`") # noqa: E501
# verify the required parameter 'namespace' is set
if self.api_client.client_side_validation and ('namespace' not in local_var_params or # noqa: E501
local_var_params['namespace'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `namespace` when calling `create_namespaced_custom_object`") # noqa: E501
# verify the required parameter 'plural' is set
if self.api_client.client_side_validation and ('plural' not in local_var_params or # noqa: E501
local_var_params['plural'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `plural` when calling `create_namespaced_custom_object`") # noqa: E501
# verify the required parameter 'body' is set
if self.api_client.client_side_validation and ('body' not in local_var_params or # noqa: E501
local_var_params['body'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `body` when calling `create_namespaced_custom_object`") # noqa: E501
collection_formats = {}
path_params = {}
if 'group' in local_var_params:
path_params['group'] = local_var_params['group'] # noqa: E501
if 'version' in local_var_params:
path_params['version'] = local_var_params['version'] # noqa: E501
if 'namespace' in local_var_params:
path_params['namespace'] = local_var_params['namespace'] # noqa: E501
if 'plural' in local_var_params:
path_params['plural'] = local_var_params['plural'] # noqa: E501
query_params = []
if 'pretty' in local_var_params and local_var_params['pretty'] is not None: # noqa: E501
query_params.append(('pretty', local_var_params['pretty'])) # noqa: E501
if 'dry_run' in local_var_params and local_var_params['dry_run'] is not None: # noqa: E501
query_params.append(('dryRun', local_var_params['dry_run'])) # noqa: E501
if 'field_manager' in local_var_params and local_var_params['field_manager'] is not None: # noqa: E501
query_params.append(('fieldManager', local_var_params['field_manager'])) # noqa: E501
if 'field_validation' in local_var_params and local_var_params['field_validation'] is not None: # noqa: E501
query_params.append(('fieldValidation', local_var_params['field_validation'])) # noqa: E501
header_params = {}
form_params = []
local_var_files = {}
body_params = None
if 'body' in local_var_params:
body_params = local_var_params['body']
# HTTP header `Accept`
header_params['Accept'] = self.api_client.select_header_accept(
['application/json']) # noqa: E501
# Authentication setting
auth_settings = ['BearerToken'] # noqa: E501
> return self.api_client.call_api(
'/apis/{group}/{version}/namespaces/{namespace}/{plural}', 'POST',
path_params,
query_params,
header_params,
body=body_params,
post_params=form_params,
files=local_var_files,
response_type='object', # noqa: E501
auth_settings=auth_settings,
async_req=local_var_params.get('async_req'),
_return_http_data_only=local_var_params.get('_return_http_data_only'), # noqa: E501
_preload_content=local_var_params.get('_preload_content', True),
_request_timeout=local_var_params.get('_request_timeout'),
collection_formats=collection_formats)
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/custom_objects_api.py:354:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
resource_path = '/apis/{group}/{version}/namespaces/{namespace}/{plural}'
method = 'POST'
path_params = {'group': 'serving.kserve.io', 'namespace': 'kserve-ci-e2e-test', 'plural': 'inferenceservices', 'version': 'v1beta1'}
query_params = []
header_params = {'Accept': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
body = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
post_params = [], files = {}, response_type = 'object'
auth_settings = ['BearerToken'], async_req = None, _return_http_data_only = True
collection_formats = {}, _preload_content = True, _request_timeout = None
_host = None
def call_api(self, resource_path, method,
path_params=None, query_params=None, header_params=None,
body=None, post_params=None, files=None,
response_type=None, auth_settings=None, async_req=None,
_return_http_data_only=None, collection_formats=None,
_preload_content=True, _request_timeout=None, _host=None):
"""Makes the HTTP request (synchronous) and returns deserialized data.
To make an async_req request, set the async_req parameter.
:param resource_path: Path to method endpoint.
:param method: Method to call.
:param path_params: Path parameters in the url.
:param query_params: Query parameters in the url.
:param header_params: Header parameters to be
placed in the request header.
:param body: Request body.
:param post_params dict: Request post form parameters,
for `application/x-www-form-urlencoded`, `multipart/form-data`.
:param auth_settings list: Auth Settings names for the request.
:param response: Response data type.
:param files dict: key -> filename, value -> filepath,
for `multipart/form-data`.
:param async_req bool: execute request asynchronously
:param _return_http_data_only: response data without head status code
and headers
:param collection_formats: dict of collection formats for path, query,
header, and post parameters.
:param _preload_content: if False, the urllib3.HTTPResponse object will
be returned without reading/decoding response
data. Default is True.
:param _request_timeout: timeout setting for this request. If one
number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
:return:
If async_req parameter is True,
the request will be called asynchronously.
The method will return the request thread.
If parameter async_req is False or missing,
then the method will return the response directly.
"""
if not async_req:
> return self.__call_api(resource_path, method,
path_params, query_params, header_params,
body, post_params, files,
response_type, auth_settings,
_return_http_data_only, collection_formats,
_preload_content, _request_timeout, _host)
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:348:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
resource_path = '/apis/serving.kserve.io/v1beta1/namespaces/kserve-ci-e2e-test/inferenceservices'
method = 'POST'
path_params = [('group', 'serving.kserve.io'), ('version', 'v1beta1'), ('namespace', 'kserve-ci-e2e-test'), ('plural', 'inferenceservices')]
query_params = []
header_params = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
body = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'name': 'success-200-isvc-071c5',...tahub/success-200-isvc:odh-pr-1326', 'name': 'kserve-container', 'resources': {'limits': {...}, 'requests': {...}}}]}}}
post_params = [], files = {}, response_type = 'object'
auth_settings = ['BearerToken'], _return_http_data_only = True
collection_formats = {}, _preload_content = True, _request_timeout = None
_host = None
def __call_api(
self, resource_path, method, path_params=None,
query_params=None, header_params=None, body=None, post_params=None,
files=None, response_type=None, auth_settings=None,
_return_http_data_only=None, collection_formats=None,
_preload_content=True, _request_timeout=None, _host=None):
config = self.configuration
# header parameters
header_params = header_params or {}
header_params.update(self.default_headers)
if self.cookie:
header_params['Cookie'] = self.cookie
if header_params:
header_params = self.sanitize_for_serialization(header_params)
header_params = dict(self.parameters_to_tuples(header_params,
collection_formats))
# path parameters
if path_params:
path_params = self.sanitize_for_serialization(path_params)
path_params = self.parameters_to_tuples(path_params,
collection_formats)
for k, v in path_params:
# specified safe chars, encode everything
resource_path = resource_path.replace(
'{%s}' % k,
quote(str(v), safe=config.safe_chars_for_path_param)
)
# query parameters
if query_params:
query_params = self.sanitize_for_serialization(query_params)
query_params = self.parameters_to_tuples(query_params,
collection_formats)
# post parameters
if post_params or files:
post_params = post_params if post_params else []
post_params = self.sanitize_for_serialization(post_params)
post_params = self.parameters_to_tuples(post_params,
collection_formats)
post_params.extend(self.files_parameters(files))
# auth setting
self.update_params_for_auth(header_params, query_params, auth_settings)
# body
if body:
body = self.sanitize_for_serialization(body)
# request url
if _host is None:
url = self.configuration.host + resource_path
else:
# use server/host defined in path or operation instead
url = _host + resource_path
# perform request and return response
> response_data = self.request(
method, url, query_params=query_params, headers=header_params,
post_params=post_params, body=body,
_preload_content=_preload_content,
_request_timeout=_request_timeout)
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:180:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
method = 'POST'
url = 'https://a9b56e18e9e4c45dcaeda4bb15f69bb8-9f16fbcdf636ba83.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1beta1/namespaces/kserve-ci-e2e-test/inferenceservices'
query_params = []
headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
post_params = []
body = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'name': 'success-200-isvc-071c5',...tahub/success-200-isvc:odh-pr-1326', 'name': 'kserve-container', 'resources': {'limits': {...}, 'requests': {...}}}]}}}
_preload_content = True, _request_timeout = None
def request(self, method, url, query_params=None, headers=None,
post_params=None, body=None, _preload_content=True,
_request_timeout=None):
"""Makes the HTTP request using RESTClient."""
if method == "GET":
return self.rest_client.GET(url,
query_params=query_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
headers=headers)
elif method == "HEAD":
return self.rest_client.HEAD(url,
query_params=query_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
headers=headers)
elif method == "OPTIONS":
return self.rest_client.OPTIONS(url,
query_params=query_params,
headers=headers,
_preload_content=_preload_content,
_request_timeout=_request_timeout)
elif method == "POST":
> return self.rest_client.POST(url,
query_params=query_params,
headers=headers,
post_params=post_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:391:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
url = 'https://a9b56e18e9e4c45dcaeda4bb15f69bb8-9f16fbcdf636ba83.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1beta1/namespaces/kserve-ci-e2e-test/inferenceservices'
headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
query_params = [], post_params = []
body = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'name': 'success-200-isvc-071c5',...tahub/success-200-isvc:odh-pr-1326', 'name': 'kserve-container', 'resources': {'limits': {...}, 'requests': {...}}}]}}}
_preload_content = True, _request_timeout = None
def POST(self, url, headers=None, query_params=None, post_params=None,
body=None, _preload_content=True, _request_timeout=None):
> return self.request("POST", url,
headers=headers,
query_params=query_params,
post_params=post_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
body=body)
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py:279:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
method = 'POST'
url = 'https://a9b56e18e9e4c45dcaeda4bb15f69bb8-9f16fbcdf636ba83.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1beta1/namespaces/kserve-ci-e2e-test/inferenceservices'
query_params = []
headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
body = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'name': 'success-200-isvc-071c5',...tahub/success-200-isvc:odh-pr-1326', 'name': 'kserve-container', 'resources': {'limits': {...}, 'requests': {...}}}]}}}
post_params = {}, _preload_content = True, _request_timeout = None
def request(self, method, url, query_params=None, headers=None,
body=None, post_params=None, _preload_content=True,
_request_timeout=None):
"""Perform requests.
:param method: http request method
:param url: http request url
:param query_params: query parameters in the url
:param headers: http request headers
:param body: request json body, for `application/json`
:param post_params: request post parameters,
`application/x-www-form-urlencoded`
and `multipart/form-data`
:param _preload_content: if False, the urllib3.HTTPResponse object will
be returned without reading/decoding response
data. Default is True.
:param _request_timeout: timeout setting for this request. If one
number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
"""
method = method.upper()
assert method in ['GET', 'HEAD', 'DELETE', 'POST', 'PUT',
'PATCH', 'OPTIONS']
if post_params and body:
raise ApiValueError(
"body parameter cannot be used with post_params parameter."
)
post_params = post_params or {}
headers = headers or {}
timeout = None
if _request_timeout:
if isinstance(_request_timeout, (int, ) if six.PY3 else (int, long)): # noqa: E501,F821
timeout = urllib3.Timeout(total=_request_timeout)
elif (isinstance(_request_timeout, tuple) and
len(_request_timeout) == 2):
timeout = urllib3.Timeout(
connect=_request_timeout[0], read=_request_timeout[1])
if 'Content-Type' not in headers:
headers['Content-Type'] = 'application/json'
try:
# For `POST`, `PUT`, `PATCH`, `OPTIONS`, `DELETE`
if method in ['POST', 'PUT', 'PATCH', 'OPTIONS', 'DELETE']:
if query_params:
url += '?' + urlencode(query_params)
if (re.search('json', headers['Content-Type'], re.IGNORECASE) or
headers['Content-Type'] == 'application/apply-patch+yaml'):
if headers['Content-Type'] == 'application/json-patch+json':
if not isinstance(body, list):
headers['Content-Type'] = \
'application/strategic-merge-patch+json'
request_body = None
if body is not None:
request_body = json.dumps(body)
r = self.pool_manager.request(
method, url,
body=request_body,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
elif headers['Content-Type'] == 'application/x-www-form-urlencoded': # noqa: E501
r = self.pool_manager.request(
method, url,
fields=post_params,
encode_multipart=False,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
elif headers['Content-Type'] == 'multipart/form-data':
# must del headers['Content-Type'], or the correct
# Content-Type which generated by urllib3 will be
# overwritten.
del headers['Content-Type']
r = self.pool_manager.request(
method, url,
fields=post_params,
encode_multipart=True,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
# Pass a `string` parameter directly in the body to support
# other content types than Json when `body` argument is
# provided in serialized form
elif isinstance(body, str) or isinstance(body, bytes):
request_body = body
r = self.pool_manager.request(
method, url,
body=request_body,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
else:
# Cannot generate the request from given parameters
msg = """Cannot prepare a request message for provided
arguments. Please check that your arguments match
declared content type."""
raise ApiException(status=0, reason=msg)
# For `GET`, `HEAD`
else:
r = self.pool_manager.request(method, url,
fields=query_params,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
except urllib3.exceptions.SSLError as e:
msg = "{0}\n{1}".format(type(e).__name__, str(e))
raise ApiException(status=0, reason=msg)
if _preload_content:
r = RESTResponse(r)
# In the python 3, the response.data is bytes.
# we need to decode it to string.
if six.PY3:
r.data = r.data.decode('utf8')
# log response body
logger.debug("response body: %s", r.data)
if not 200 <= r.status <= 299:
> raise ApiException(http_resp=r)
E kubernetes.client.exceptions.ApiException: (500)
E Reason: Internal Server Error
E HTTP response headers: HTTPHeaderDict({'Audit-Id': 'aa73626f-5211-4d2e-9cdb-297c29becb5b', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Strict-Transport-Security': 'max-age=31536000; includeSubDomains; preload', 'X-Kubernetes-Pf-Flowschema-Uid': '224291cc-bc8a-47e1-b358-99f64d47f314', 'X-Kubernetes-Pf-Prioritylevel-Uid': '53379352-efa0-40c7-b71a-ad76894c832e', 'Date': 'Thu, 16 Apr 2026 20:28:43 GMT', 'Content-Length': '769'})
E HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: failed calling webhook \"inferenceservice.kserve-webhook-server.defaulter\": failed to call webhook: Post \"https://kserve-webhook-server-service.kserve.svc:443/mutate-serving-kserve-io-v1beta1-inferenceservice?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","reason":"InternalError","details":{"causes":[{"message":"failed calling webhook \"inferenceservice.kserve-webhook-server.defaulter\": failed to call webhook: Post \"https://kserve-webhook-server-service.kserve.svc:443/mutate-serving-kserve-io-v1beta1-inferenceservice?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\""}]},"code":500}
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py:238: ApiException
During handling of the above exception, another exception occurred:
rest_v1_client =
@pytest.mark.graph
@pytest.mark.kourier
@pytest.mark.asyncio(scope="session")
async def test_ig_scenario8(rest_v1_client):
"""
Scenario: Ensemble graph with 3 steps, where 2 steps are soft and 1 step is hard and returns non-200
Expectation: Since HARD step will return non-200, so IG will return that step's output as IG's output
"""
logger.info("Starting test test_ig_scenario8")
suffix = str(uuid.uuid4())[1:6]
success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test(
suffix
)
logger.info(f"success_isvc_name is {success_isvc_name}")
logger.info(f"error_isvc_name is {error_isvc_name}")
kserve_client = KServeClient(
config_file=os.environ.get("KUBECONFIG", "~/.kube/config")
)
> kserve_client.create(success_isvc)
graph/test_inference_graph.py:758:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
inferenceservice = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
namespace = 'kserve-ci-e2e-test', watch = False, timeout_seconds = 600
def create(
self, inferenceservice, namespace=None, watch=False, timeout_seconds=600
): # pylint:disable=inconsistent-return-statements
"""
Create the inference service
:param inferenceservice: inference service object
:param namespace: defaults to current or default namespace
:param watch: True to watch the created service until timeout elapsed or status is ready
:param timeout_seconds: timeout seconds for watch, default to 600s
:return: created inference service
"""
version = inferenceservice.api_version.split("/")[1]
if namespace is None:
namespace = utils.get_isvc_namespace(inferenceservice)
try:
outputs = self.api_instance.create_namespaced_custom_object(
constants.KSERVE_GROUP,
version,
namespace,
constants.KSERVE_PLURAL_INFERENCESERVICE,
inferenceservice,
)
except client.rest.ApiException as e:
> raise RuntimeError(
"Exception when calling CustomObjectsApi->create_namespaced_custom_object:\
%s\n"
% e
)
E RuntimeError: Exception when calling CustomObjectsApi->create_namespaced_custom_object: (500)
E Reason: Internal Server Error
E HTTP response headers: HTTPHeaderDict({'Audit-Id': 'aa73626f-5211-4d2e-9cdb-297c29becb5b', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Strict-Transport-Security': 'max-age=31536000; includeSubDomains; preload', 'X-Kubernetes-Pf-Flowschema-Uid': '224291cc-bc8a-47e1-b358-99f64d47f314', 'X-Kubernetes-Pf-Prioritylevel-Uid': '53379352-efa0-40c7-b71a-ad76894c832e', 'Date': 'Thu, 16 Apr 2026 20:28:43 GMT', 'Content-Length': '769'})
E HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: failed calling webhook \"inferenceservice.kserve-webhook-server.defaulter\": failed to call webhook: Post \"https://kserve-webhook-server-service.kserve.svc:443/mutate-serving-kserve-io-v1beta1-inferenceservice?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","reason":"InternalError","details":{"causes":[{"message":"failed calling webhook \"inferenceservice.kserve-webhook-server.defaulter\": failed to call webhook: Post \"https://kserve-webhook-server-service.kserve.svc:443/mutate-serving-kserve-io-v1beta1-inferenceservice?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\""}]},"code":500}
../../python/kserve/kserve/api/kserve_client.py:153: RuntimeError
------------------------------ Captured log call -------------------------------
INFO kserve.trace:test_inference_graph.py:747 Starting test test_ig_scenario8
INFO kserve.trace:test_inference_graph.py:207 SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
INFO kserve.trace:test_inference_graph.py:208 ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
INFO kserve.trace:test_inference_graph.py:752 success_isvc_name is success-200-isvc-071c5
INFO kserve.trace:test_inference_graph.py:753 error_isvc_name is error-404-isvc-071c5
______________________________ test_ig_scenario9 _______________________________
[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python
self =
inferenceservice = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
namespace = 'kserve-ci-e2e-test', watch = False, timeout_seconds = 600
def create(
self, inferenceservice, namespace=None, watch=False, timeout_seconds=600
): # pylint:disable=inconsistent-return-statements
"""
Create the inference service
:param inferenceservice: inference service object
:param namespace: defaults to current or default namespace
:param watch: True to watch the created service until timeout elapsed or status is ready
:param timeout_seconds: timeout seconds for watch, default to 600s
:return: created inference service
"""
version = inferenceservice.api_version.split("/")[1]
if namespace is None:
namespace = utils.get_isvc_namespace(inferenceservice)
try:
> outputs = self.api_instance.create_namespaced_custom_object(
constants.KSERVE_GROUP,
version,
namespace,
constants.KSERVE_PLURAL_INFERENCESERVICE,
inferenceservice,
)
../../python/kserve/kserve/api/kserve_client.py:145:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
group = 'serving.kserve.io', version = 'v1beta1'
namespace = 'kserve-ci-e2e-test', plural = 'inferenceservices'
body = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
kwargs = {'_return_http_data_only': True}
def create_namespaced_custom_object(self, group, version, namespace, plural, body, **kwargs): # noqa: E501
"""create_namespaced_custom_object # noqa: E501
Creates a namespace scoped Custom object # noqa: E501
This method makes a synchronous HTTP request by default. To make an
asynchronous HTTP request, please pass async_req=True
>>> thread = api.create_namespaced_custom_object(group, version, namespace, plural, body, async_req=True)
>>> result = thread.get()
:param async_req bool: execute request asynchronously
:param str group: The custom resource's group name (required)
:param str version: The custom resource's version (required)
:param str namespace: The custom resource's namespace (required)
:param str plural: The custom resource's plural name. For TPRs this would be lowercase plural kind. (required)
:param object body: The JSON schema of the Resource to create. (required)
:param str pretty: If 'true', then the output is pretty printed.
:param str dry_run: When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed
:param str field_manager: fieldManager is a name associated with the actor or entity that is making these changes. The value must be less than or 128 characters long, and only contain printable characters, as defined by https://golang.org/pkg/unicode/#IsPrint.
:param str field_validation: fieldValidation instructs the server on how to handle objects in the request (POST/PUT/PATCH) containing unknown or duplicate fields. Valid values are: - Ignore: This will ignore any unknown fields that are silently dropped from the object, and will ignore all but the last duplicate field that the decoder encounters. This is the default behavior prior to v1.23. - Warn: This will send a warning via the standard warning response header for each unknown field that is dropped from the object, and for each duplicate field that is encountered. The request will still succeed if there are no other errors, and will only persist the last of any duplicate fields. This is the default in v1.23+ - Strict: This will fail the request with a BadRequest error if any unknown fields would be dropped from the object, or if any duplicate fields are present. The error returned from the server will contain all unknown and duplicate fields encountered. (optional)
:param _preload_content: if False, the urllib3.HTTPResponse object will
be returned without reading/decoding response
data. Default is True.
:param _request_timeout: timeout setting for this request. If one
number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
:return: object
If the method is called asynchronously,
returns the request thread.
"""
kwargs['_return_http_data_only'] = True
> return self.create_namespaced_custom_object_with_http_info(group, version, namespace, plural, body, **kwargs) # noqa: E501
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/custom_objects_api.py:231:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
group = 'serving.kserve.io', version = 'v1beta1'
namespace = 'kserve-ci-e2e-test', plural = 'inferenceservices'
body = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
kwargs = {'_return_http_data_only': True}
local_var_params = {'_return_http_data_only': True, 'all_params': ['group', 'version', 'namespace', 'plural', 'body', 'pretty', ...], 'au... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}, ...}
all_params = ['group', 'version', 'namespace', 'plural', 'body', 'pretty', ...]
key = '_return_http_data_only', val = True, collection_formats = {}
path_params = {'group': 'serving.kserve.io', 'namespace': 'kserve-ci-e2e-test', 'plural': 'inferenceservices', 'version': 'v1beta1'}
query_params = []
def create_namespaced_custom_object_with_http_info(self, group, version, namespace, plural, body, **kwargs): # noqa: E501
"""create_namespaced_custom_object # noqa: E501
Creates a namespace scoped Custom object # noqa: E501
This method makes a synchronous HTTP request by default. To make an
asynchronous HTTP request, please pass async_req=True
>>> thread = api.create_namespaced_custom_object_with_http_info(group, version, namespace, plural, body, async_req=True)
>>> result = thread.get()
:param async_req bool: execute request asynchronously
:param str group: The custom resource's group name (required)
:param str version: The custom resource's version (required)
:param str namespace: The custom resource's namespace (required)
:param str plural: The custom resource's plural name. For TPRs this would be lowercase plural kind. (required)
:param object body: The JSON schema of the Resource to create. (required)
:param str pretty: If 'true', then the output is pretty printed.
:param str dry_run: When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed
:param str field_manager: fieldManager is a name associated with the actor or entity that is making these changes. The value must be less than or 128 characters long, and only contain printable characters, as defined by https://golang.org/pkg/unicode/#IsPrint.
:param str field_validation: fieldValidation instructs the server on how to handle objects in the request (POST/PUT/PATCH) containing unknown or duplicate fields. Valid values are: - Ignore: This will ignore any unknown fields that are silently dropped from the object, and will ignore all but the last duplicate field that the decoder encounters. This is the default behavior prior to v1.23. - Warn: This will send a warning via the standard warning response header for each unknown field that is dropped from the object, and for each duplicate field that is encountered. The request will still succeed if there are no other errors, and will only persist the last of any duplicate fields. This is the default in v1.23+ - Strict: This will fail the request with a BadRequest error if any unknown fields would be dropped from the object, or if any duplicate fields are present. The error returned from the server will contain all unknown and duplicate fields encountered. (optional)
:param _return_http_data_only: response data without head status code
and headers
:param _preload_content: if False, the urllib3.HTTPResponse object will
be returned without reading/decoding response
data. Default is True.
:param _request_timeout: timeout setting for this request. If one
number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
:return: tuple(object, status_code(int), headers(HTTPHeaderDict))
If the method is called asynchronously,
returns the request thread.
"""
local_var_params = locals()
all_params = [
'group',
'version',
'namespace',
'plural',
'body',
'pretty',
'dry_run',
'field_manager',
'field_validation'
]
all_params.extend(
[
'async_req',
'_return_http_data_only',
'_preload_content',
'_request_timeout'
]
)
for key, val in six.iteritems(local_var_params['kwargs']):
if key not in all_params:
raise ApiTypeError(
"Got an unexpected keyword argument '%s'"
" to method create_namespaced_custom_object" % key
)
local_var_params[key] = val
del local_var_params['kwargs']
# verify the required parameter 'group' is set
if self.api_client.client_side_validation and ('group' not in local_var_params or # noqa: E501
local_var_params['group'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `group` when calling `create_namespaced_custom_object`") # noqa: E501
# verify the required parameter 'version' is set
if self.api_client.client_side_validation and ('version' not in local_var_params or # noqa: E501
local_var_params['version'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `version` when calling `create_namespaced_custom_object`") # noqa: E501
# verify the required parameter 'namespace' is set
if self.api_client.client_side_validation and ('namespace' not in local_var_params or # noqa: E501
local_var_params['namespace'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `namespace` when calling `create_namespaced_custom_object`") # noqa: E501
# verify the required parameter 'plural' is set
if self.api_client.client_side_validation and ('plural' not in local_var_params or # noqa: E501
local_var_params['plural'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `plural` when calling `create_namespaced_custom_object`") # noqa: E501
# verify the required parameter 'body' is set
if self.api_client.client_side_validation and ('body' not in local_var_params or # noqa: E501
local_var_params['body'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `body` when calling `create_namespaced_custom_object`") # noqa: E501
collection_formats = {}
path_params = {}
if 'group' in local_var_params:
path_params['group'] = local_var_params['group'] # noqa: E501
if 'version' in local_var_params:
path_params['version'] = local_var_params['version'] # noqa: E501
if 'namespace' in local_var_params:
path_params['namespace'] = local_var_params['namespace'] # noqa: E501
if 'plural' in local_var_params:
path_params['plural'] = local_var_params['plural'] # noqa: E501
query_params = []
if 'pretty' in local_var_params and local_var_params['pretty'] is not None: # noqa: E501
query_params.append(('pretty', local_var_params['pretty'])) # noqa: E501
if 'dry_run' in local_var_params and local_var_params['dry_run'] is not None: # noqa: E501
query_params.append(('dryRun', local_var_params['dry_run'])) # noqa: E501
if 'field_manager' in local_var_params and local_var_params['field_manager'] is not None: # noqa: E501
query_params.append(('fieldManager', local_var_params['field_manager'])) # noqa: E501
if 'field_validation' in local_var_params and local_var_params['field_validation'] is not None: # noqa: E501
query_params.append(('fieldValidation', local_var_params['field_validation'])) # noqa: E501
header_params = {}
form_params = []
local_var_files = {}
body_params = None
if 'body' in local_var_params:
body_params = local_var_params['body']
# HTTP header `Accept`
header_params['Accept'] = self.api_client.select_header_accept(
['application/json']) # noqa: E501
# Authentication setting
auth_settings = ['BearerToken'] # noqa: E501
> return self.api_client.call_api(
'/apis/{group}/{version}/namespaces/{namespace}/{plural}', 'POST',
path_params,
query_params,
header_params,
body=body_params,
post_params=form_params,
files=local_var_files,
response_type='object', # noqa: E501
auth_settings=auth_settings,
async_req=local_var_params.get('async_req'),
_return_http_data_only=local_var_params.get('_return_http_data_only'), # noqa: E501
_preload_content=local_var_params.get('_preload_content', True),
_request_timeout=local_var_params.get('_request_timeout'),
collection_formats=collection_formats)
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/custom_objects_api.py:354:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
resource_path = '/apis/{group}/{version}/namespaces/{namespace}/{plural}'
method = 'POST'
path_params = {'group': 'serving.kserve.io', 'namespace': 'kserve-ci-e2e-test', 'plural': 'inferenceservices', 'version': 'v1beta1'}
query_params = []
header_params = {'Accept': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
body = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
post_params = [], files = {}, response_type = 'object'
auth_settings = ['BearerToken'], async_req = None, _return_http_data_only = True
collection_formats = {}, _preload_content = True, _request_timeout = None
_host = None
def call_api(self, resource_path, method,
path_params=None, query_params=None, header_params=None,
body=None, post_params=None, files=None,
response_type=None, auth_settings=None, async_req=None,
_return_http_data_only=None, collection_formats=None,
_preload_content=True, _request_timeout=None, _host=None):
"""Makes the HTTP request (synchronous) and returns deserialized data.
To make an async_req request, set the async_req parameter.
:param resource_path: Path to method endpoint.
:param method: Method to call.
:param path_params: Path parameters in the url.
:param query_params: Query parameters in the url.
:param header_params: Header parameters to be
placed in the request header.
:param body: Request body.
:param post_params dict: Request post form parameters,
for `application/x-www-form-urlencoded`, `multipart/form-data`.
:param auth_settings list: Auth Settings names for the request.
:param response: Response data type.
:param files dict: key -> filename, value -> filepath,
for `multipart/form-data`.
:param async_req bool: execute request asynchronously
:param _return_http_data_only: response data without head status code
and headers
:param collection_formats: dict of collection formats for path, query,
header, and post parameters.
:param _preload_content: if False, the urllib3.HTTPResponse object will
be returned without reading/decoding response
data. Default is True.
:param _request_timeout: timeout setting for this request. If one
number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
:return:
If async_req parameter is True,
the request will be called asynchronously.
The method will return the request thread.
If parameter async_req is False or missing,
then the method will return the response directly.
"""
if not async_req:
> return self.__call_api(resource_path, method,
path_params, query_params, header_params,
body, post_params, files,
response_type, auth_settings,
_return_http_data_only, collection_formats,
_preload_content, _request_timeout, _host)
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:348:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
resource_path = '/apis/serving.kserve.io/v1beta1/namespaces/kserve-ci-e2e-test/inferenceservices'
method = 'POST'
path_params = [('group', 'serving.kserve.io'), ('version', 'v1beta1'), ('namespace', 'kserve-ci-e2e-test'), ('plural', 'inferenceservices')]
query_params = []
header_params = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
body = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'name': 'success-200-isvc-0f5dd',...tahub/success-200-isvc:odh-pr-1326', 'name': 'kserve-container', 'resources': {'limits': {...}, 'requests': {...}}}]}}}
post_params = [], files = {}, response_type = 'object'
auth_settings = ['BearerToken'], _return_http_data_only = True
collection_formats = {}, _preload_content = True, _request_timeout = None
_host = None
def __call_api(
self, resource_path, method, path_params=None,
query_params=None, header_params=None, body=None, post_params=None,
files=None, response_type=None, auth_settings=None,
_return_http_data_only=None, collection_formats=None,
_preload_content=True, _request_timeout=None, _host=None):
config = self.configuration
# header parameters
header_params = header_params or {}
header_params.update(self.default_headers)
if self.cookie:
header_params['Cookie'] = self.cookie
if header_params:
header_params = self.sanitize_for_serialization(header_params)
header_params = dict(self.parameters_to_tuples(header_params,
collection_formats))
# path parameters
if path_params:
path_params = self.sanitize_for_serialization(path_params)
path_params = self.parameters_to_tuples(path_params,
collection_formats)
for k, v in path_params:
# specified safe chars, encode everything
resource_path = resource_path.replace(
'{%s}' % k,
quote(str(v), safe=config.safe_chars_for_path_param)
)
# query parameters
if query_params:
query_params = self.sanitize_for_serialization(query_params)
query_params = self.parameters_to_tuples(query_params,
collection_formats)
# post parameters
if post_params or files:
post_params = post_params if post_params else []
post_params = self.sanitize_for_serialization(post_params)
post_params = self.parameters_to_tuples(post_params,
collection_formats)
post_params.extend(self.files_parameters(files))
# auth setting
self.update_params_for_auth(header_params, query_params, auth_settings)
# body
if body:
body = self.sanitize_for_serialization(body)
# request url
if _host is None:
url = self.configuration.host + resource_path
else:
# use server/host defined in path or operation instead
url = _host + resource_path
# perform request and return response
> response_data = self.request(
method, url, query_params=query_params, headers=header_params,
post_params=post_params, body=body,
_preload_content=_preload_content,
_request_timeout=_request_timeout)
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:180:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
method = 'POST'
url = 'https://a9b56e18e9e4c45dcaeda4bb15f69bb8-9f16fbcdf636ba83.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1beta1/namespaces/kserve-ci-e2e-test/inferenceservices'
query_params = []
headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
post_params = []
body = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'name': 'success-200-isvc-0f5dd',...tahub/success-200-isvc:odh-pr-1326', 'name': 'kserve-container', 'resources': {'limits': {...}, 'requests': {...}}}]}}}
_preload_content = True, _request_timeout = None
def request(self, method, url, query_params=None, headers=None,
post_params=None, body=None, _preload_content=True,
_request_timeout=None):
"""Makes the HTTP request using RESTClient."""
if method == "GET":
return self.rest_client.GET(url,
query_params=query_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
headers=headers)
elif method == "HEAD":
return self.rest_client.HEAD(url,
query_params=query_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
headers=headers)
elif method == "OPTIONS":
return self.rest_client.OPTIONS(url,
query_params=query_params,
headers=headers,
_preload_content=_preload_content,
_request_timeout=_request_timeout)
elif method == "POST":
> return self.rest_client.POST(url,
query_params=query_params,
headers=headers,
post_params=post_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:391:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
url = 'https://a9b56e18e9e4c45dcaeda4bb15f69bb8-9f16fbcdf636ba83.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1beta1/namespaces/kserve-ci-e2e-test/inferenceservices'
headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
query_params = [], post_params = []
body = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'name': 'success-200-isvc-0f5dd',...tahub/success-200-isvc:odh-pr-1326', 'name': 'kserve-container', 'resources': {'limits': {...}, 'requests': {...}}}]}}}
_preload_content = True, _request_timeout = None
def POST(self, url, headers=None, query_params=None, post_params=None,
body=None, _preload_content=True, _request_timeout=None):
> return self.request("POST", url,
headers=headers,
query_params=query_params,
post_params=post_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
body=body)
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py:279:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
method = 'POST'
url = 'https://a9b56e18e9e4c45dcaeda4bb15f69bb8-9f16fbcdf636ba83.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1beta1/namespaces/kserve-ci-e2e-test/inferenceservices'
query_params = []
headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
body = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'name': 'success-200-isvc-0f5dd',...tahub/success-200-isvc:odh-pr-1326', 'name': 'kserve-container', 'resources': {'limits': {...}, 'requests': {...}}}]}}}
post_params = {}, _preload_content = True, _request_timeout = None
def request(self, method, url, query_params=None, headers=None,
body=None, post_params=None, _preload_content=True,
_request_timeout=None):
"""Perform requests.
:param method: http request method
:param url: http request url
:param query_params: query parameters in the url
:param headers: http request headers
:param body: request json body, for `application/json`
:param post_params: request post parameters,
`application/x-www-form-urlencoded`
and `multipart/form-data`
:param _preload_content: if False, the urllib3.HTTPResponse object will
be returned without reading/decoding response
data. Default is True.
:param _request_timeout: timeout setting for this request. If one
number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
"""
method = method.upper()
assert method in ['GET', 'HEAD', 'DELETE', 'POST', 'PUT',
'PATCH', 'OPTIONS']
if post_params and body:
raise ApiValueError(
"body parameter cannot be used with post_params parameter."
)
post_params = post_params or {}
headers = headers or {}
timeout = None
if _request_timeout:
if isinstance(_request_timeout, (int, ) if six.PY3 else (int, long)): # noqa: E501,F821
timeout = urllib3.Timeout(total=_request_timeout)
elif (isinstance(_request_timeout, tuple) and
len(_request_timeout) == 2):
timeout = urllib3.Timeout(
connect=_request_timeout[0], read=_request_timeout[1])
if 'Content-Type' not in headers:
headers['Content-Type'] = 'application/json'
try:
# For `POST`, `PUT`, `PATCH`, `OPTIONS`, `DELETE`
if method in ['POST', 'PUT', 'PATCH', 'OPTIONS', 'DELETE']:
if query_params:
url += '?' + urlencode(query_params)
if (re.search('json', headers['Content-Type'], re.IGNORECASE) or
headers['Content-Type'] == 'application/apply-patch+yaml'):
if headers['Content-Type'] == 'application/json-patch+json':
if not isinstance(body, list):
headers['Content-Type'] = \
'application/strategic-merge-patch+json'
request_body = None
if body is not None:
request_body = json.dumps(body)
r = self.pool_manager.request(
method, url,
body=request_body,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
elif headers['Content-Type'] == 'application/x-www-form-urlencoded': # noqa: E501
r = self.pool_manager.request(
method, url,
fields=post_params,
encode_multipart=False,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
elif headers['Content-Type'] == 'multipart/form-data':
# must del headers['Content-Type'], or the correct
# Content-Type which generated by urllib3 will be
# overwritten.
del headers['Content-Type']
r = self.pool_manager.request(
method, url,
fields=post_params,
encode_multipart=True,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
# Pass a `string` parameter directly in the body to support
# other content types than Json when `body` argument is
# provided in serialized form
elif isinstance(body, str) or isinstance(body, bytes):
request_body = body
r = self.pool_manager.request(
method, url,
body=request_body,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
else:
# Cannot generate the request from given parameters
msg = """Cannot prepare a request message for provided
arguments. Please check that your arguments match
declared content type."""
raise ApiException(status=0, reason=msg)
# For `GET`, `HEAD`
else:
r = self.pool_manager.request(method, url,
fields=query_params,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
except urllib3.exceptions.SSLError as e:
msg = "{0}\n{1}".format(type(e).__name__, str(e))
raise ApiException(status=0, reason=msg)
if _preload_content:
r = RESTResponse(r)
# In the python 3, the response.data is bytes.
# we need to decode it to string.
if six.PY3:
r.data = r.data.decode('utf8')
# log response body
logger.debug("response body: %s", r.data)
if not 200 <= r.status <= 299:
> raise ApiException(http_resp=r)
E kubernetes.client.exceptions.ApiException: (500)
E Reason: Internal Server Error
E HTTP response headers: HTTPHeaderDict({'Audit-Id': 'c183490f-2c32-497e-bc91-39a47ba5aab4', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Strict-Transport-Security': 'max-age=31536000; includeSubDomains; preload', 'X-Kubernetes-Pf-Flowschema-Uid': '224291cc-bc8a-47e1-b358-99f64d47f314', 'X-Kubernetes-Pf-Prioritylevel-Uid': '53379352-efa0-40c7-b71a-ad76894c832e', 'Date': 'Thu, 16 Apr 2026 20:28:44 GMT', 'Content-Length': '769'})
E HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: failed calling webhook \"inferenceservice.kserve-webhook-server.defaulter\": failed to call webhook: Post \"https://kserve-webhook-server-service.kserve.svc:443/mutate-serving-kserve-io-v1beta1-inferenceservice?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","reason":"InternalError","details":{"causes":[{"message":"failed calling webhook \"inferenceservice.kserve-webhook-server.defaulter\": failed to call webhook: Post \"https://kserve-webhook-server-service.kserve.svc:443/mutate-serving-kserve-io-v1beta1-inferenceservice?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\""}]},"code":500}
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py:238: ApiException
During handling of the above exception, another exception occurred:
rest_v1_client =
@pytest.mark.graph
@pytest.mark.kourier
@pytest.mark.asyncio(scope="session")
async def test_ig_scenario9(rest_v1_client):
"""
Scenario: Splitter graph where a match would happen for error node and then error would return but IG will continue
execution and call the next step in the flow as error step will be a soft dependency.
Expectation: IG will return response of success_isvc.
"""
logger.info("Starting test test_ig_scenario9")
suffix = str(uuid.uuid4())[1:6]
success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test(
suffix
)
logger.info(f"success_isvc_name is {success_isvc_name}")
logger.info(f"error_isvc_name is {error_isvc_name}")
kserve_client = KServeClient(
config_file=os.environ.get("KUBECONFIG", "~/.kube/config")
)
> kserve_client.create(success_isvc)
graph/test_inference_graph.py:824:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
inferenceservice = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
namespace = 'kserve-ci-e2e-test', watch = False, timeout_seconds = 600
def create(
self, inferenceservice, namespace=None, watch=False, timeout_seconds=600
): # pylint:disable=inconsistent-return-statements
"""
Create the inference service
:param inferenceservice: inference service object
:param namespace: defaults to current or default namespace
:param watch: True to watch the created service until timeout elapsed or status is ready
:param timeout_seconds: timeout seconds for watch, default to 600s
:return: created inference service
"""
version = inferenceservice.api_version.split("/")[1]
if namespace is None:
namespace = utils.get_isvc_namespace(inferenceservice)
try:
outputs = self.api_instance.create_namespaced_custom_object(
constants.KSERVE_GROUP,
version,
namespace,
constants.KSERVE_PLURAL_INFERENCESERVICE,
inferenceservice,
)
except client.rest.ApiException as e:
> raise RuntimeError(
"Exception when calling CustomObjectsApi->create_namespaced_custom_object:\
%s\n"
% e
)
E RuntimeError: Exception when calling CustomObjectsApi->create_namespaced_custom_object: (500)
E Reason: Internal Server Error
E HTTP response headers: HTTPHeaderDict({'Audit-Id': 'c183490f-2c32-497e-bc91-39a47ba5aab4', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Strict-Transport-Security': 'max-age=31536000; includeSubDomains; preload', 'X-Kubernetes-Pf-Flowschema-Uid': '224291cc-bc8a-47e1-b358-99f64d47f314', 'X-Kubernetes-Pf-Prioritylevel-Uid': '53379352-efa0-40c7-b71a-ad76894c832e', 'Date': 'Thu, 16 Apr 2026 20:28:44 GMT', 'Content-Length': '769'})
E HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: failed calling webhook \"inferenceservice.kserve-webhook-server.defaulter\": failed to call webhook: Post \"https://kserve-webhook-server-service.kserve.svc:443/mutate-serving-kserve-io-v1beta1-inferenceservice?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","reason":"InternalError","details":{"causes":[{"message":"failed calling webhook \"inferenceservice.kserve-webhook-server.defaulter\": failed to call webhook: Post \"https://kserve-webhook-server-service.kserve.svc:443/mutate-serving-kserve-io-v1beta1-inferenceservice?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\""}]},"code":500}
../../python/kserve/kserve/api/kserve_client.py:153: RuntimeError
------------------------------ Captured log call -------------------------------
INFO kserve.trace:test_inference_graph.py:813 Starting test test_ig_scenario9
INFO kserve.trace:test_inference_graph.py:207 SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
INFO kserve.trace:test_inference_graph.py:208 ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
INFO kserve.trace:test_inference_graph.py:818 success_isvc_name is success-200-isvc-0f5dd
INFO kserve.trace:test_inference_graph.py:819 error_isvc_name is error-404-isvc-0f5dd
______________________________ test_ig_scenario10 ______________________________
[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python
self =
inferenceservice = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
namespace = 'kserve-ci-e2e-test', watch = False, timeout_seconds = 600
def create(
self, inferenceservice, namespace=None, watch=False, timeout_seconds=600
): # pylint:disable=inconsistent-return-statements
"""
Create the inference service
:param inferenceservice: inference service object
:param namespace: defaults to current or default namespace
:param watch: True to watch the created service until timeout elapsed or status is ready
:param timeout_seconds: timeout seconds for watch, default to 600s
:return: created inference service
"""
version = inferenceservice.api_version.split("/")[1]
if namespace is None:
namespace = utils.get_isvc_namespace(inferenceservice)
try:
> outputs = self.api_instance.create_namespaced_custom_object(
constants.KSERVE_GROUP,
version,
namespace,
constants.KSERVE_PLURAL_INFERENCESERVICE,
inferenceservice,
)
../../python/kserve/kserve/api/kserve_client.py:145:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
group = 'serving.kserve.io', version = 'v1beta1'
namespace = 'kserve-ci-e2e-test', plural = 'inferenceservices'
body = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
kwargs = {'_return_http_data_only': True}
def create_namespaced_custom_object(self, group, version, namespace, plural, body, **kwargs): # noqa: E501
"""create_namespaced_custom_object # noqa: E501
Creates a namespace scoped Custom object # noqa: E501
This method makes a synchronous HTTP request by default. To make an
asynchronous HTTP request, please pass async_req=True
>>> thread = api.create_namespaced_custom_object(group, version, namespace, plural, body, async_req=True)
>>> result = thread.get()
:param async_req bool: execute request asynchronously
:param str group: The custom resource's group name (required)
:param str version: The custom resource's version (required)
:param str namespace: The custom resource's namespace (required)
:param str plural: The custom resource's plural name. For TPRs this would be lowercase plural kind. (required)
:param object body: The JSON schema of the Resource to create. (required)
:param str pretty: If 'true', then the output is pretty printed.
:param str dry_run: When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed
:param str field_manager: fieldManager is a name associated with the actor or entity that is making these changes. The value must be less than or 128 characters long, and only contain printable characters, as defined by https://golang.org/pkg/unicode/#IsPrint.
:param str field_validation: fieldValidation instructs the server on how to handle objects in the request (POST/PUT/PATCH) containing unknown or duplicate fields. Valid values are: - Ignore: This will ignore any unknown fields that are silently dropped from the object, and will ignore all but the last duplicate field that the decoder encounters. This is the default behavior prior to v1.23. - Warn: This will send a warning via the standard warning response header for each unknown field that is dropped from the object, and for each duplicate field that is encountered. The request will still succeed if there are no other errors, and will only persist the last of any duplicate fields. This is the default in v1.23+ - Strict: This will fail the request with a BadRequest error if any unknown fields would be dropped from the object, or if any duplicate fields are present. The error returned from the server will contain all unknown and duplicate fields encountered. (optional)
:param _preload_content: if False, the urllib3.HTTPResponse object will
be returned without reading/decoding response
data. Default is True.
:param _request_timeout: timeout setting for this request. If one
number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
:return: object
If the method is called asynchronously,
returns the request thread.
"""
kwargs['_return_http_data_only'] = True
> return self.create_namespaced_custom_object_with_http_info(group, version, namespace, plural, body, **kwargs) # noqa: E501
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/custom_objects_api.py:231:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
group = 'serving.kserve.io', version = 'v1beta1'
namespace = 'kserve-ci-e2e-test', plural = 'inferenceservices'
body = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
kwargs = {'_return_http_data_only': True}
local_var_params = {'_return_http_data_only': True, 'all_params': ['group', 'version', 'namespace', 'plural', 'body', 'pretty', ...], 'au... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}, ...}
all_params = ['group', 'version', 'namespace', 'plural', 'body', 'pretty', ...]
key = '_return_http_data_only', val = True, collection_formats = {}
path_params = {'group': 'serving.kserve.io', 'namespace': 'kserve-ci-e2e-test', 'plural': 'inferenceservices', 'version': 'v1beta1'}
query_params = []
def create_namespaced_custom_object_with_http_info(self, group, version, namespace, plural, body, **kwargs): # noqa: E501
"""create_namespaced_custom_object # noqa: E501
Creates a namespace scoped Custom object # noqa: E501
This method makes a synchronous HTTP request by default. To make an
asynchronous HTTP request, please pass async_req=True
>>> thread = api.create_namespaced_custom_object_with_http_info(group, version, namespace, plural, body, async_req=True)
>>> result = thread.get()
:param async_req bool: execute request asynchronously
:param str group: The custom resource's group name (required)
:param str version: The custom resource's version (required)
:param str namespace: The custom resource's namespace (required)
:param str plural: The custom resource's plural name. For TPRs this would be lowercase plural kind. (required)
:param object body: The JSON schema of the Resource to create. (required)
:param str pretty: If 'true', then the output is pretty printed.
:param str dry_run: When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed
:param str field_manager: fieldManager is a name associated with the actor or entity that is making these changes. The value must be less than or 128 characters long, and only contain printable characters, as defined by https://golang.org/pkg/unicode/#IsPrint.
:param str field_validation: fieldValidation instructs the server on how to handle objects in the request (POST/PUT/PATCH) containing unknown or duplicate fields. Valid values are: - Ignore: This will ignore any unknown fields that are silently dropped from the object, and will ignore all but the last duplicate field that the decoder encounters. This is the default behavior prior to v1.23. - Warn: This will send a warning via the standard warning response header for each unknown field that is dropped from the object, and for each duplicate field that is encountered. The request will still succeed if there are no other errors, and will only persist the last of any duplicate fields. This is the default in v1.23+ - Strict: This will fail the request with a BadRequest error if any unknown fields would be dropped from the object, or if any duplicate fields are present. The error returned from the server will contain all unknown and duplicate fields encountered. (optional)
:param _return_http_data_only: response data without head status code
and headers
:param _preload_content: if False, the urllib3.HTTPResponse object will
be returned without reading/decoding response
data. Default is True.
:param _request_timeout: timeout setting for this request. If one
number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
:return: tuple(object, status_code(int), headers(HTTPHeaderDict))
If the method is called asynchronously,
returns the request thread.
"""
local_var_params = locals()
all_params = [
'group',
'version',
'namespace',
'plural',
'body',
'pretty',
'dry_run',
'field_manager',
'field_validation'
]
all_params.extend(
[
'async_req',
'_return_http_data_only',
'_preload_content',
'_request_timeout'
]
)
for key, val in six.iteritems(local_var_params['kwargs']):
if key not in all_params:
raise ApiTypeError(
"Got an unexpected keyword argument '%s'"
" to method create_namespaced_custom_object" % key
)
local_var_params[key] = val
del local_var_params['kwargs']
# verify the required parameter 'group' is set
if self.api_client.client_side_validation and ('group' not in local_var_params or # noqa: E501
local_var_params['group'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `group` when calling `create_namespaced_custom_object`") # noqa: E501
# verify the required parameter 'version' is set
if self.api_client.client_side_validation and ('version' not in local_var_params or # noqa: E501
local_var_params['version'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `version` when calling `create_namespaced_custom_object`") # noqa: E501
# verify the required parameter 'namespace' is set
if self.api_client.client_side_validation and ('namespace' not in local_var_params or # noqa: E501
local_var_params['namespace'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `namespace` when calling `create_namespaced_custom_object`") # noqa: E501
# verify the required parameter 'plural' is set
if self.api_client.client_side_validation and ('plural' not in local_var_params or # noqa: E501
local_var_params['plural'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `plural` when calling `create_namespaced_custom_object`") # noqa: E501
# verify the required parameter 'body' is set
if self.api_client.client_side_validation and ('body' not in local_var_params or # noqa: E501
local_var_params['body'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `body` when calling `create_namespaced_custom_object`") # noqa: E501
collection_formats = {}
path_params = {}
if 'group' in local_var_params:
path_params['group'] = local_var_params['group'] # noqa: E501
if 'version' in local_var_params:
path_params['version'] = local_var_params['version'] # noqa: E501
if 'namespace' in local_var_params:
path_params['namespace'] = local_var_params['namespace'] # noqa: E501
if 'plural' in local_var_params:
path_params['plural'] = local_var_params['plural'] # noqa: E501
query_params = []
if 'pretty' in local_var_params and local_var_params['pretty'] is not None: # noqa: E501
query_params.append(('pretty', local_var_params['pretty'])) # noqa: E501
if 'dry_run' in local_var_params and local_var_params['dry_run'] is not None: # noqa: E501
query_params.append(('dryRun', local_var_params['dry_run'])) # noqa: E501
if 'field_manager' in local_var_params and local_var_params['field_manager'] is not None: # noqa: E501
query_params.append(('fieldManager', local_var_params['field_manager'])) # noqa: E501
if 'field_validation' in local_var_params and local_var_params['field_validation'] is not None: # noqa: E501
query_params.append(('fieldValidation', local_var_params['field_validation'])) # noqa: E501
header_params = {}
form_params = []
local_var_files = {}
body_params = None
if 'body' in local_var_params:
body_params = local_var_params['body']
# HTTP header `Accept`
header_params['Accept'] = self.api_client.select_header_accept(
['application/json']) # noqa: E501
# Authentication setting
auth_settings = ['BearerToken'] # noqa: E501
> return self.api_client.call_api(
'/apis/{group}/{version}/namespaces/{namespace}/{plural}', 'POST',
path_params,
query_params,
header_params,
body=body_params,
post_params=form_params,
files=local_var_files,
response_type='object', # noqa: E501
auth_settings=auth_settings,
async_req=local_var_params.get('async_req'),
_return_http_data_only=local_var_params.get('_return_http_data_only'), # noqa: E501
_preload_content=local_var_params.get('_preload_content', True),
_request_timeout=local_var_params.get('_request_timeout'),
collection_formats=collection_formats)
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/custom_objects_api.py:354:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
resource_path = '/apis/{group}/{version}/namespaces/{namespace}/{plural}'
method = 'POST'
path_params = {'group': 'serving.kserve.io', 'namespace': 'kserve-ci-e2e-test', 'plural': 'inferenceservices', 'version': 'v1beta1'}
query_params = []
header_params = {'Accept': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
body = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
post_params = [], files = {}, response_type = 'object'
auth_settings = ['BearerToken'], async_req = None, _return_http_data_only = True
collection_formats = {}, _preload_content = True, _request_timeout = None
_host = None
def call_api(self, resource_path, method,
path_params=None, query_params=None, header_params=None,
body=None, post_params=None, files=None,
response_type=None, auth_settings=None, async_req=None,
_return_http_data_only=None, collection_formats=None,
_preload_content=True, _request_timeout=None, _host=None):
"""Makes the HTTP request (synchronous) and returns deserialized data.
To make an async_req request, set the async_req parameter.
:param resource_path: Path to method endpoint.
:param method: Method to call.
:param path_params: Path parameters in the url.
:param query_params: Query parameters in the url.
:param header_params: Header parameters to be
placed in the request header.
:param body: Request body.
:param post_params dict: Request post form parameters,
for `application/x-www-form-urlencoded`, `multipart/form-data`.
:param auth_settings list: Auth Settings names for the request.
:param response: Response data type.
:param files dict: key -> filename, value -> filepath,
for `multipart/form-data`.
:param async_req bool: execute request asynchronously
:param _return_http_data_only: response data without head status code
and headers
:param collection_formats: dict of collection formats for path, query,
header, and post parameters.
:param _preload_content: if False, the urllib3.HTTPResponse object will
be returned without reading/decoding response
data. Default is True.
:param _request_timeout: timeout setting for this request. If one
number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
:return:
If async_req parameter is True,
the request will be called asynchronously.
The method will return the request thread.
If parameter async_req is False or missing,
then the method will return the response directly.
"""
if not async_req:
> return self.__call_api(resource_path, method,
path_params, query_params, header_params,
body, post_params, files,
response_type, auth_settings,
_return_http_data_only, collection_formats,
_preload_content, _request_timeout, _host)
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:348:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
resource_path = '/apis/serving.kserve.io/v1beta1/namespaces/kserve-ci-e2e-test/inferenceservices'
method = 'POST'
path_params = [('group', 'serving.kserve.io'), ('version', 'v1beta1'), ('namespace', 'kserve-ci-e2e-test'), ('plural', 'inferenceservices')]
query_params = []
header_params = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
body = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'name': 'success-200-isvc-c640d',...tahub/success-200-isvc:odh-pr-1326', 'name': 'kserve-container', 'resources': {'limits': {...}, 'requests': {...}}}]}}}
post_params = [], files = {}, response_type = 'object'
auth_settings = ['BearerToken'], _return_http_data_only = True
collection_formats = {}, _preload_content = True, _request_timeout = None
_host = None
def __call_api(
self, resource_path, method, path_params=None,
query_params=None, header_params=None, body=None, post_params=None,
files=None, response_type=None, auth_settings=None,
_return_http_data_only=None, collection_formats=None,
_preload_content=True, _request_timeout=None, _host=None):
config = self.configuration
# header parameters
header_params = header_params or {}
header_params.update(self.default_headers)
if self.cookie:
header_params['Cookie'] = self.cookie
if header_params:
header_params = self.sanitize_for_serialization(header_params)
header_params = dict(self.parameters_to_tuples(header_params,
collection_formats))
# path parameters
if path_params:
path_params = self.sanitize_for_serialization(path_params)
path_params = self.parameters_to_tuples(path_params,
collection_formats)
for k, v in path_params:
# specified safe chars, encode everything
resource_path = resource_path.replace(
'{%s}' % k,
quote(str(v), safe=config.safe_chars_for_path_param)
)
# query parameters
if query_params:
query_params = self.sanitize_for_serialization(query_params)
query_params = self.parameters_to_tuples(query_params,
collection_formats)
# post parameters
if post_params or files:
post_params = post_params if post_params else []
post_params = self.sanitize_for_serialization(post_params)
post_params = self.parameters_to_tuples(post_params,
collection_formats)
post_params.extend(self.files_parameters(files))
# auth setting
self.update_params_for_auth(header_params, query_params, auth_settings)
# body
if body:
body = self.sanitize_for_serialization(body)
# request url
if _host is None:
url = self.configuration.host + resource_path
else:
# use server/host defined in path or operation instead
url = _host + resource_path
# perform request and return response
> response_data = self.request(
method, url, query_params=query_params, headers=header_params,
post_params=post_params, body=body,
_preload_content=_preload_content,
_request_timeout=_request_timeout)
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:180:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
method = 'POST'
url = 'https://a9b56e18e9e4c45dcaeda4bb15f69bb8-9f16fbcdf636ba83.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1beta1/namespaces/kserve-ci-e2e-test/inferenceservices'
query_params = []
headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
post_params = []
body = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'name': 'success-200-isvc-c640d',...tahub/success-200-isvc:odh-pr-1326', 'name': 'kserve-container', 'resources': {'limits': {...}, 'requests': {...}}}]}}}
_preload_content = True, _request_timeout = None
def request(self, method, url, query_params=None, headers=None,
post_params=None, body=None, _preload_content=True,
_request_timeout=None):
"""Makes the HTTP request using RESTClient."""
if method == "GET":
return self.rest_client.GET(url,
query_params=query_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
headers=headers)
elif method == "HEAD":
return self.rest_client.HEAD(url,
query_params=query_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
headers=headers)
elif method == "OPTIONS":
return self.rest_client.OPTIONS(url,
query_params=query_params,
headers=headers,
_preload_content=_preload_content,
_request_timeout=_request_timeout)
elif method == "POST":
> return self.rest_client.POST(url,
query_params=query_params,
headers=headers,
post_params=post_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:391:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
url = 'https://a9b56e18e9e4c45dcaeda4bb15f69bb8-9f16fbcdf636ba83.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1beta1/namespaces/kserve-ci-e2e-test/inferenceservices'
headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
query_params = [], post_params = []
body = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'name': 'success-200-isvc-c640d',...tahub/success-200-isvc:odh-pr-1326', 'name': 'kserve-container', 'resources': {'limits': {...}, 'requests': {...}}}]}}}
_preload_content = True, _request_timeout = None
def POST(self, url, headers=None, query_params=None, post_params=None,
body=None, _preload_content=True, _request_timeout=None):
> return self.request("POST", url,
headers=headers,
query_params=query_params,
post_params=post_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
body=body)
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py:279:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
method = 'POST'
url = 'https://a9b56e18e9e4c45dcaeda4bb15f69bb8-9f16fbcdf636ba83.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1beta1/namespaces/kserve-ci-e2e-test/inferenceservices'
query_params = []
headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
body = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'name': 'success-200-isvc-c640d',...tahub/success-200-isvc:odh-pr-1326', 'name': 'kserve-container', 'resources': {'limits': {...}, 'requests': {...}}}]}}}
post_params = {}, _preload_content = True, _request_timeout = None
def request(self, method, url, query_params=None, headers=None,
body=None, post_params=None, _preload_content=True,
_request_timeout=None):
"""Perform requests.
:param method: http request method
:param url: http request url
:param query_params: query parameters in the url
:param headers: http request headers
:param body: request json body, for `application/json`
:param post_params: request post parameters,
`application/x-www-form-urlencoded`
and `multipart/form-data`
:param _preload_content: if False, the urllib3.HTTPResponse object will
be returned without reading/decoding response
data. Default is True.
:param _request_timeout: timeout setting for this request. If one
number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
"""
method = method.upper()
assert method in ['GET', 'HEAD', 'DELETE', 'POST', 'PUT',
'PATCH', 'OPTIONS']
if post_params and body:
raise ApiValueError(
"body parameter cannot be used with post_params parameter."
)
post_params = post_params or {}
headers = headers or {}
timeout = None
if _request_timeout:
if isinstance(_request_timeout, (int, ) if six.PY3 else (int, long)): # noqa: E501,F821
timeout = urllib3.Timeout(total=_request_timeout)
elif (isinstance(_request_timeout, tuple) and
len(_request_timeout) == 2):
timeout = urllib3.Timeout(
connect=_request_timeout[0], read=_request_timeout[1])
if 'Content-Type' not in headers:
headers['Content-Type'] = 'application/json'
try:
# For `POST`, `PUT`, `PATCH`, `OPTIONS`, `DELETE`
if method in ['POST', 'PUT', 'PATCH', 'OPTIONS', 'DELETE']:
if query_params:
url += '?' + urlencode(query_params)
if (re.search('json', headers['Content-Type'], re.IGNORECASE) or
headers['Content-Type'] == 'application/apply-patch+yaml'):
if headers['Content-Type'] == 'application/json-patch+json':
if not isinstance(body, list):
headers['Content-Type'] = \
'application/strategic-merge-patch+json'
request_body = None
if body is not None:
request_body = json.dumps(body)
r = self.pool_manager.request(
method, url,
body=request_body,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
elif headers['Content-Type'] == 'application/x-www-form-urlencoded': # noqa: E501
r = self.pool_manager.request(
method, url,
fields=post_params,
encode_multipart=False,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
elif headers['Content-Type'] == 'multipart/form-data':
# must del headers['Content-Type'], or the correct
# Content-Type which generated by urllib3 will be
# overwritten.
del headers['Content-Type']
r = self.pool_manager.request(
method, url,
fields=post_params,
encode_multipart=True,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
# Pass a `string` parameter directly in the body to support
# other content types than Json when `body` argument is
# provided in serialized form
elif isinstance(body, str) or isinstance(body, bytes):
request_body = body
r = self.pool_manager.request(
method, url,
body=request_body,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
else:
# Cannot generate the request from given parameters
msg = """Cannot prepare a request message for provided
arguments. Please check that your arguments match
declared content type."""
raise ApiException(status=0, reason=msg)
# For `GET`, `HEAD`
else:
r = self.pool_manager.request(method, url,
fields=query_params,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
except urllib3.exceptions.SSLError as e:
msg = "{0}\n{1}".format(type(e).__name__, str(e))
raise ApiException(status=0, reason=msg)
if _preload_content:
r = RESTResponse(r)
# In the python 3, the response.data is bytes.
# we need to decode it to string.
if six.PY3:
r.data = r.data.decode('utf8')
# log response body
logger.debug("response body: %s", r.data)
if not 200 <= r.status <= 299:
> raise ApiException(http_resp=r)
E kubernetes.client.exceptions.ApiException: (500)
E Reason: Internal Server Error
E HTTP response headers: HTTPHeaderDict({'Audit-Id': 'd9f1d7de-254a-4e09-a547-0f0a44096c90', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Strict-Transport-Security': 'max-age=31536000; includeSubDomains; preload', 'X-Kubernetes-Pf-Flowschema-Uid': '224291cc-bc8a-47e1-b358-99f64d47f314', 'X-Kubernetes-Pf-Prioritylevel-Uid': '53379352-efa0-40c7-b71a-ad76894c832e', 'Date': 'Thu, 16 Apr 2026 20:28:44 GMT', 'Content-Length': '769'})
E HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: failed calling webhook \"inferenceservice.kserve-webhook-server.defaulter\": failed to call webhook: Post \"https://kserve-webhook-server-service.kserve.svc:443/mutate-serving-kserve-io-v1beta1-inferenceservice?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","reason":"InternalError","details":{"causes":[{"message":"failed calling webhook \"inferenceservice.kserve-webhook-server.defaulter\": failed to call webhook: Post \"https://kserve-webhook-server-service.kserve.svc:443/mutate-serving-kserve-io-v1beta1-inferenceservice?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\""}]},"code":500}
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py:238: ApiException
During handling of the above exception, another exception occurred:
rest_v1_client =
@pytest.mark.graph
@pytest.mark.kourier
@pytest.mark.asyncio(scope="session")
async def test_ig_scenario10(rest_v1_client):
"""
Scenario: Splitter graph where a match would happen for error node and then error would return and IG will NOT
continue execution and call the next step in the flow as error step will be a HARD dependency.
Expectation: IG will return response of success_isvc.
"""
logger.info("Starting test test_ig_scenario10")
suffix = str(uuid.uuid4())[1:6]
success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test(
suffix
)
logger.info(f"success_isvc_name is {success_isvc_name}")
logger.info(f"error_isvc_name is {error_isvc_name}")
kserve_client = KServeClient(
config_file=os.environ.get("KUBECONFIG", "~/.kube/config")
)
> kserve_client.create(success_isvc)
graph/test_inference_graph.py:886:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
inferenceservice = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
namespace = 'kserve-ci-e2e-test', watch = False, timeout_seconds = 600
def create(
self, inferenceservice, namespace=None, watch=False, timeout_seconds=600
): # pylint:disable=inconsistent-return-statements
"""
Create the inference service
:param inferenceservice: inference service object
:param namespace: defaults to current or default namespace
:param watch: True to watch the created service until timeout elapsed or status is ready
:param timeout_seconds: timeout seconds for watch, default to 600s
:return: created inference service
"""
version = inferenceservice.api_version.split("/")[1]
if namespace is None:
namespace = utils.get_isvc_namespace(inferenceservice)
try:
outputs = self.api_instance.create_namespaced_custom_object(
constants.KSERVE_GROUP,
version,
namespace,
constants.KSERVE_PLURAL_INFERENCESERVICE,
inferenceservice,
)
except client.rest.ApiException as e:
> raise RuntimeError(
"Exception when calling CustomObjectsApi->create_namespaced_custom_object:\
%s\n"
% e
)
E RuntimeError: Exception when calling CustomObjectsApi->create_namespaced_custom_object: (500)
E Reason: Internal Server Error
E HTTP response headers: HTTPHeaderDict({'Audit-Id': 'd9f1d7de-254a-4e09-a547-0f0a44096c90', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Strict-Transport-Security': 'max-age=31536000; includeSubDomains; preload', 'X-Kubernetes-Pf-Flowschema-Uid': '224291cc-bc8a-47e1-b358-99f64d47f314', 'X-Kubernetes-Pf-Prioritylevel-Uid': '53379352-efa0-40c7-b71a-ad76894c832e', 'Date': 'Thu, 16 Apr 2026 20:28:44 GMT', 'Content-Length': '769'})
E HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: failed calling webhook \"inferenceservice.kserve-webhook-server.defaulter\": failed to call webhook: Post \"https://kserve-webhook-server-service.kserve.svc:443/mutate-serving-kserve-io-v1beta1-inferenceservice?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","reason":"InternalError","details":{"causes":[{"message":"failed calling webhook \"inferenceservice.kserve-webhook-server.defaulter\": failed to call webhook: Post \"https://kserve-webhook-server-service.kserve.svc:443/mutate-serving-kserve-io-v1beta1-inferenceservice?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\""}]},"code":500}
../../python/kserve/kserve/api/kserve_client.py:153: RuntimeError
------------------------------ Captured log call -------------------------------
INFO kserve.trace:test_inference_graph.py:875 Starting test test_ig_scenario10
INFO kserve.trace:test_inference_graph.py:207 SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
INFO kserve.trace:test_inference_graph.py:208 ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
INFO kserve.trace:test_inference_graph.py:880 success_isvc_name is success-200-isvc-c640d
INFO kserve.trace:test_inference_graph.py:881 error_isvc_name is error-404-isvc-c640d
______________________________ test_ig_scenario4 _______________________________
[gw1] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python
self =
inferenceservice = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
namespace = 'kserve-ci-e2e-test', watch = False, timeout_seconds = 600
def create(
self, inferenceservice, namespace=None, watch=False, timeout_seconds=600
): # pylint:disable=inconsistent-return-statements
"""
Create the inference service
:param inferenceservice: inference service object
:param namespace: defaults to current or default namespace
:param watch: True to watch the created service until timeout elapsed or status is ready
:param timeout_seconds: timeout seconds for watch, default to 600s
:return: created inference service
"""
version = inferenceservice.api_version.split("/")[1]
if namespace is None:
namespace = utils.get_isvc_namespace(inferenceservice)
try:
> outputs = self.api_instance.create_namespaced_custom_object(
constants.KSERVE_GROUP,
version,
namespace,
constants.KSERVE_PLURAL_INFERENCESERVICE,
inferenceservice,
)
../../python/kserve/kserve/api/kserve_client.py:145:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
group = 'serving.kserve.io', version = 'v1beta1'
namespace = 'kserve-ci-e2e-test', plural = 'inferenceservices'
body = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
kwargs = {'_return_http_data_only': True}
def create_namespaced_custom_object(self, group, version, namespace, plural, body, **kwargs): # noqa: E501
"""create_namespaced_custom_object # noqa: E501
Creates a namespace scoped Custom object # noqa: E501
This method makes a synchronous HTTP request by default. To make an
asynchronous HTTP request, please pass async_req=True
>>> thread = api.create_namespaced_custom_object(group, version, namespace, plural, body, async_req=True)
>>> result = thread.get()
:param async_req bool: execute request asynchronously
:param str group: The custom resource's group name (required)
:param str version: The custom resource's version (required)
:param str namespace: The custom resource's namespace (required)
:param str plural: The custom resource's plural name. For TPRs this would be lowercase plural kind. (required)
:param object body: The JSON schema of the Resource to create. (required)
:param str pretty: If 'true', then the output is pretty printed.
:param str dry_run: When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed
:param str field_manager: fieldManager is a name associated with the actor or entity that is making these changes. The value must be less than or 128 characters long, and only contain printable characters, as defined by https://golang.org/pkg/unicode/#IsPrint.
:param str field_validation: fieldValidation instructs the server on how to handle objects in the request (POST/PUT/PATCH) containing unknown or duplicate fields. Valid values are: - Ignore: This will ignore any unknown fields that are silently dropped from the object, and will ignore all but the last duplicate field that the decoder encounters. This is the default behavior prior to v1.23. - Warn: This will send a warning via the standard warning response header for each unknown field that is dropped from the object, and for each duplicate field that is encountered. The request will still succeed if there are no other errors, and will only persist the last of any duplicate fields. This is the default in v1.23+ - Strict: This will fail the request with a BadRequest error if any unknown fields would be dropped from the object, or if any duplicate fields are present. The error returned from the server will contain all unknown and duplicate fields encountered. (optional)
:param _preload_content: if False, the urllib3.HTTPResponse object will
be returned without reading/decoding response
data. Default is True.
:param _request_timeout: timeout setting for this request. If one
number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
:return: object
If the method is called asynchronously,
returns the request thread.
"""
kwargs['_return_http_data_only'] = True
> return self.create_namespaced_custom_object_with_http_info(group, version, namespace, plural, body, **kwargs) # noqa: E501
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/custom_objects_api.py:231:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
group = 'serving.kserve.io', version = 'v1beta1'
namespace = 'kserve-ci-e2e-test', plural = 'inferenceservices'
body = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
kwargs = {'_return_http_data_only': True}
local_var_params = {'_return_http_data_only': True, 'all_params': ['group', 'version', 'namespace', 'plural', 'body', 'pretty', ...], 'au... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}, ...}
all_params = ['group', 'version', 'namespace', 'plural', 'body', 'pretty', ...]
key = '_return_http_data_only', val = True, collection_formats = {}
path_params = {'group': 'serving.kserve.io', 'namespace': 'kserve-ci-e2e-test', 'plural': 'inferenceservices', 'version': 'v1beta1'}
query_params = []
def create_namespaced_custom_object_with_http_info(self, group, version, namespace, plural, body, **kwargs): # noqa: E501
"""create_namespaced_custom_object # noqa: E501
Creates a namespace scoped Custom object # noqa: E501
This method makes a synchronous HTTP request by default. To make an
asynchronous HTTP request, please pass async_req=True
>>> thread = api.create_namespaced_custom_object_with_http_info(group, version, namespace, plural, body, async_req=True)
>>> result = thread.get()
:param async_req bool: execute request asynchronously
:param str group: The custom resource's group name (required)
:param str version: The custom resource's version (required)
:param str namespace: The custom resource's namespace (required)
:param str plural: The custom resource's plural name. For TPRs this would be lowercase plural kind. (required)
:param object body: The JSON schema of the Resource to create. (required)
:param str pretty: If 'true', then the output is pretty printed.
:param str dry_run: When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed
:param str field_manager: fieldManager is a name associated with the actor or entity that is making these changes. The value must be less than or 128 characters long, and only contain printable characters, as defined by https://golang.org/pkg/unicode/#IsPrint.
:param str field_validation: fieldValidation instructs the server on how to handle objects in the request (POST/PUT/PATCH) containing unknown or duplicate fields. Valid values are: - Ignore: This will ignore any unknown fields that are silently dropped from the object, and will ignore all but the last duplicate field that the decoder encounters. This is the default behavior prior to v1.23. - Warn: This will send a warning via the standard warning response header for each unknown field that is dropped from the object, and for each duplicate field that is encountered. The request will still succeed if there are no other errors, and will only persist the last of any duplicate fields. This is the default in v1.23+ - Strict: This will fail the request with a BadRequest error if any unknown fields would be dropped from the object, or if any duplicate fields are present. The error returned from the server will contain all unknown and duplicate fields encountered. (optional)
:param _return_http_data_only: response data without head status code
and headers
:param _preload_content: if False, the urllib3.HTTPResponse object will
be returned without reading/decoding response
data. Default is True.
:param _request_timeout: timeout setting for this request. If one
number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
:return: tuple(object, status_code(int), headers(HTTPHeaderDict))
If the method is called asynchronously,
returns the request thread.
"""
local_var_params = locals()
all_params = [
'group',
'version',
'namespace',
'plural',
'body',
'pretty',
'dry_run',
'field_manager',
'field_validation'
]
all_params.extend(
[
'async_req',
'_return_http_data_only',
'_preload_content',
'_request_timeout'
]
)
for key, val in six.iteritems(local_var_params['kwargs']):
if key not in all_params:
raise ApiTypeError(
"Got an unexpected keyword argument '%s'"
" to method create_namespaced_custom_object" % key
)
local_var_params[key] = val
del local_var_params['kwargs']
# verify the required parameter 'group' is set
if self.api_client.client_side_validation and ('group' not in local_var_params or # noqa: E501
local_var_params['group'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `group` when calling `create_namespaced_custom_object`") # noqa: E501
# verify the required parameter 'version' is set
if self.api_client.client_side_validation and ('version' not in local_var_params or # noqa: E501
local_var_params['version'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `version` when calling `create_namespaced_custom_object`") # noqa: E501
# verify the required parameter 'namespace' is set
if self.api_client.client_side_validation and ('namespace' not in local_var_params or # noqa: E501
local_var_params['namespace'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `namespace` when calling `create_namespaced_custom_object`") # noqa: E501
# verify the required parameter 'plural' is set
if self.api_client.client_side_validation and ('plural' not in local_var_params or # noqa: E501
local_var_params['plural'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `plural` when calling `create_namespaced_custom_object`") # noqa: E501
# verify the required parameter 'body' is set
if self.api_client.client_side_validation and ('body' not in local_var_params or # noqa: E501
local_var_params['body'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `body` when calling `create_namespaced_custom_object`") # noqa: E501
collection_formats = {}
path_params = {}
if 'group' in local_var_params:
path_params['group'] = local_var_params['group'] # noqa: E501
if 'version' in local_var_params:
path_params['version'] = local_var_params['version'] # noqa: E501
if 'namespace' in local_var_params:
path_params['namespace'] = local_var_params['namespace'] # noqa: E501
if 'plural' in local_var_params:
path_params['plural'] = local_var_params['plural'] # noqa: E501
query_params = []
if 'pretty' in local_var_params and local_var_params['pretty'] is not None: # noqa: E501
query_params.append(('pretty', local_var_params['pretty'])) # noqa: E501
if 'dry_run' in local_var_params and local_var_params['dry_run'] is not None: # noqa: E501
query_params.append(('dryRun', local_var_params['dry_run'])) # noqa: E501
if 'field_manager' in local_var_params and local_var_params['field_manager'] is not None: # noqa: E501
query_params.append(('fieldManager', local_var_params['field_manager'])) # noqa: E501
if 'field_validation' in local_var_params and local_var_params['field_validation'] is not None: # noqa: E501
query_params.append(('fieldValidation', local_var_params['field_validation'])) # noqa: E501
header_params = {}
form_params = []
local_var_files = {}
body_params = None
if 'body' in local_var_params:
body_params = local_var_params['body']
# HTTP header `Accept`
header_params['Accept'] = self.api_client.select_header_accept(
['application/json']) # noqa: E501
# Authentication setting
auth_settings = ['BearerToken'] # noqa: E501
> return self.api_client.call_api(
'/apis/{group}/{version}/namespaces/{namespace}/{plural}', 'POST',
path_params,
query_params,
header_params,
body=body_params,
post_params=form_params,
files=local_var_files,
response_type='object', # noqa: E501
auth_settings=auth_settings,
async_req=local_var_params.get('async_req'),
_return_http_data_only=local_var_params.get('_return_http_data_only'), # noqa: E501
_preload_content=local_var_params.get('_preload_content', True),
_request_timeout=local_var_params.get('_request_timeout'),
collection_formats=collection_formats)
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/custom_objects_api.py:354:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
resource_path = '/apis/{group}/{version}/namespaces/{namespace}/{plural}'
method = 'POST'
path_params = {'group': 'serving.kserve.io', 'namespace': 'kserve-ci-e2e-test', 'plural': 'inferenceservices', 'version': 'v1beta1'}
query_params = []
header_params = {'Accept': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
body = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
post_params = [], files = {}, response_type = 'object'
auth_settings = ['BearerToken'], async_req = None, _return_http_data_only = True
collection_formats = {}, _preload_content = True, _request_timeout = None
_host = None
def call_api(self, resource_path, method,
path_params=None, query_params=None, header_params=None,
body=None, post_params=None, files=None,
response_type=None, auth_settings=None, async_req=None,
_return_http_data_only=None, collection_formats=None,
_preload_content=True, _request_timeout=None, _host=None):
"""Makes the HTTP request (synchronous) and returns deserialized data.
To make an async_req request, set the async_req parameter.
:param resource_path: Path to method endpoint.
:param method: Method to call.
:param path_params: Path parameters in the url.
:param query_params: Query parameters in the url.
:param header_params: Header parameters to be
placed in the request header.
:param body: Request body.
:param post_params dict: Request post form parameters,
for `application/x-www-form-urlencoded`, `multipart/form-data`.
:param auth_settings list: Auth Settings names for the request.
:param response: Response data type.
:param files dict: key -> filename, value -> filepath,
for `multipart/form-data`.
:param async_req bool: execute request asynchronously
:param _return_http_data_only: response data without head status code
and headers
:param collection_formats: dict of collection formats for path, query,
header, and post parameters.
:param _preload_content: if False, the urllib3.HTTPResponse object will
be returned without reading/decoding response
data. Default is True.
:param _request_timeout: timeout setting for this request. If one
number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
:return:
If async_req parameter is True,
the request will be called asynchronously.
The method will return the request thread.
If parameter async_req is False or missing,
then the method will return the response directly.
"""
if not async_req:
> return self.__call_api(resource_path, method,
path_params, query_params, header_params,
body, post_params, files,
response_type, auth_settings,
_return_http_data_only, collection_formats,
_preload_content, _request_timeout, _host)
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:348:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
resource_path = '/apis/serving.kserve.io/v1beta1/namespaces/kserve-ci-e2e-test/inferenceservices'
method = 'POST'
path_params = [('group', 'serving.kserve.io'), ('version', 'v1beta1'), ('namespace', 'kserve-ci-e2e-test'), ('plural', 'inferenceservices')]
query_params = []
header_params = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
body = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'name': 'success-200-isvc-d7402',...tahub/success-200-isvc:odh-pr-1326', 'name': 'kserve-container', 'resources': {'limits': {...}, 'requests': {...}}}]}}}
post_params = [], files = {}, response_type = 'object'
auth_settings = ['BearerToken'], _return_http_data_only = True
collection_formats = {}, _preload_content = True, _request_timeout = None
_host = None
def __call_api(
self, resource_path, method, path_params=None,
query_params=None, header_params=None, body=None, post_params=None,
files=None, response_type=None, auth_settings=None,
_return_http_data_only=None, collection_formats=None,
_preload_content=True, _request_timeout=None, _host=None):
config = self.configuration
# header parameters
header_params = header_params or {}
header_params.update(self.default_headers)
if self.cookie:
header_params['Cookie'] = self.cookie
if header_params:
header_params = self.sanitize_for_serialization(header_params)
header_params = dict(self.parameters_to_tuples(header_params,
collection_formats))
# path parameters
if path_params:
path_params = self.sanitize_for_serialization(path_params)
path_params = self.parameters_to_tuples(path_params,
collection_formats)
for k, v in path_params:
# specified safe chars, encode everything
resource_path = resource_path.replace(
'{%s}' % k,
quote(str(v), safe=config.safe_chars_for_path_param)
)
# query parameters
if query_params:
query_params = self.sanitize_for_serialization(query_params)
query_params = self.parameters_to_tuples(query_params,
collection_formats)
# post parameters
if post_params or files:
post_params = post_params if post_params else []
post_params = self.sanitize_for_serialization(post_params)
post_params = self.parameters_to_tuples(post_params,
collection_formats)
post_params.extend(self.files_parameters(files))
# auth setting
self.update_params_for_auth(header_params, query_params, auth_settings)
# body
if body:
body = self.sanitize_for_serialization(body)
# request url
if _host is None:
url = self.configuration.host + resource_path
else:
# use server/host defined in path or operation instead
url = _host + resource_path
# perform request and return response
> response_data = self.request(
method, url, query_params=query_params, headers=header_params,
post_params=post_params, body=body,
_preload_content=_preload_content,
_request_timeout=_request_timeout)
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:180:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
method = 'POST'
url = 'https://a9b56e18e9e4c45dcaeda4bb15f69bb8-9f16fbcdf636ba83.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1beta1/namespaces/kserve-ci-e2e-test/inferenceservices'
query_params = []
headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
post_params = []
body = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'name': 'success-200-isvc-d7402',...tahub/success-200-isvc:odh-pr-1326', 'name': 'kserve-container', 'resources': {'limits': {...}, 'requests': {...}}}]}}}
_preload_content = True, _request_timeout = None
def request(self, method, url, query_params=None, headers=None,
post_params=None, body=None, _preload_content=True,
_request_timeout=None):
"""Makes the HTTP request using RESTClient."""
if method == "GET":
return self.rest_client.GET(url,
query_params=query_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
headers=headers)
elif method == "HEAD":
return self.rest_client.HEAD(url,
query_params=query_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
headers=headers)
elif method == "OPTIONS":
return self.rest_client.OPTIONS(url,
query_params=query_params,
headers=headers,
_preload_content=_preload_content,
_request_timeout=_request_timeout)
elif method == "POST":
> return self.rest_client.POST(url,
query_params=query_params,
headers=headers,
post_params=post_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:391:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
url = 'https://a9b56e18e9e4c45dcaeda4bb15f69bb8-9f16fbcdf636ba83.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1beta1/namespaces/kserve-ci-e2e-test/inferenceservices'
headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
query_params = [], post_params = []
body = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'name': 'success-200-isvc-d7402',...tahub/success-200-isvc:odh-pr-1326', 'name': 'kserve-container', 'resources': {'limits': {...}, 'requests': {...}}}]}}}
_preload_content = True, _request_timeout = None
def POST(self, url, headers=None, query_params=None, post_params=None,
body=None, _preload_content=True, _request_timeout=None):
> return self.request("POST", url,
headers=headers,
query_params=query_params,
post_params=post_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
body=body)
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py:279:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
method = 'POST'
url = 'https://a9b56e18e9e4c45dcaeda4bb15f69bb8-9f16fbcdf636ba83.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1beta1/namespaces/kserve-ci-e2e-test/inferenceservices'
query_params = []
headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
body = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'name': 'success-200-isvc-d7402',...tahub/success-200-isvc:odh-pr-1326', 'name': 'kserve-container', 'resources': {'limits': {...}, 'requests': {...}}}]}}}
post_params = {}, _preload_content = True, _request_timeout = None
def request(self, method, url, query_params=None, headers=None,
body=None, post_params=None, _preload_content=True,
_request_timeout=None):
"""Perform requests.
:param method: http request method
:param url: http request url
:param query_params: query parameters in the url
:param headers: http request headers
:param body: request json body, for `application/json`
:param post_params: request post parameters,
`application/x-www-form-urlencoded`
and `multipart/form-data`
:param _preload_content: if False, the urllib3.HTTPResponse object will
be returned without reading/decoding response
data. Default is True.
:param _request_timeout: timeout setting for this request. If one
number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
"""
method = method.upper()
assert method in ['GET', 'HEAD', 'DELETE', 'POST', 'PUT',
'PATCH', 'OPTIONS']
if post_params and body:
raise ApiValueError(
"body parameter cannot be used with post_params parameter."
)
post_params = post_params or {}
headers = headers or {}
timeout = None
if _request_timeout:
if isinstance(_request_timeout, (int, ) if six.PY3 else (int, long)): # noqa: E501,F821
timeout = urllib3.Timeout(total=_request_timeout)
elif (isinstance(_request_timeout, tuple) and
len(_request_timeout) == 2):
timeout = urllib3.Timeout(
connect=_request_timeout[0], read=_request_timeout[1])
if 'Content-Type' not in headers:
headers['Content-Type'] = 'application/json'
try:
# For `POST`, `PUT`, `PATCH`, `OPTIONS`, `DELETE`
if method in ['POST', 'PUT', 'PATCH', 'OPTIONS', 'DELETE']:
if query_params:
url += '?' + urlencode(query_params)
if (re.search('json', headers['Content-Type'], re.IGNORECASE) or
headers['Content-Type'] == 'application/apply-patch+yaml'):
if headers['Content-Type'] == 'application/json-patch+json':
if not isinstance(body, list):
headers['Content-Type'] = \
'application/strategic-merge-patch+json'
request_body = None
if body is not None:
request_body = json.dumps(body)
r = self.pool_manager.request(
method, url,
body=request_body,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
elif headers['Content-Type'] == 'application/x-www-form-urlencoded': # noqa: E501
r = self.pool_manager.request(
method, url,
fields=post_params,
encode_multipart=False,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
elif headers['Content-Type'] == 'multipart/form-data':
# must del headers['Content-Type'], or the correct
# Content-Type which generated by urllib3 will be
# overwritten.
del headers['Content-Type']
r = self.pool_manager.request(
method, url,
fields=post_params,
encode_multipart=True,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
# Pass a `string` parameter directly in the body to support
# other content types than Json when `body` argument is
# provided in serialized form
elif isinstance(body, str) or isinstance(body, bytes):
request_body = body
r = self.pool_manager.request(
method, url,
body=request_body,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
else:
# Cannot generate the request from given parameters
msg = """Cannot prepare a request message for provided
arguments. Please check that your arguments match
declared content type."""
raise ApiException(status=0, reason=msg)
# For `GET`, `HEAD`
else:
r = self.pool_manager.request(method, url,
fields=query_params,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
except urllib3.exceptions.SSLError as e:
msg = "{0}\n{1}".format(type(e).__name__, str(e))
raise ApiException(status=0, reason=msg)
if _preload_content:
r = RESTResponse(r)
# In the python 3, the response.data is bytes.
# we need to decode it to string.
if six.PY3:
r.data = r.data.decode('utf8')
# log response body
logger.debug("response body: %s", r.data)
if not 200 <= r.status <= 299:
> raise ApiException(http_resp=r)
E kubernetes.client.exceptions.ApiException: (500)
E Reason: Internal Server Error
E HTTP response headers: HTTPHeaderDict({'Audit-Id': 'a0ab5abb-05c2-4373-aa8f-488aec4a9794', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Strict-Transport-Security': 'max-age=31536000; includeSubDomains; preload', 'X-Kubernetes-Pf-Flowschema-Uid': '224291cc-bc8a-47e1-b358-99f64d47f314', 'X-Kubernetes-Pf-Prioritylevel-Uid': '53379352-efa0-40c7-b71a-ad76894c832e', 'Date': 'Thu, 16 Apr 2026 20:28:44 GMT', 'Content-Length': '769'})
E HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: failed calling webhook \"inferenceservice.kserve-webhook-server.defaulter\": failed to call webhook: Post \"https://kserve-webhook-server-service.kserve.svc:443/mutate-serving-kserve-io-v1beta1-inferenceservice?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","reason":"InternalError","details":{"causes":[{"message":"failed calling webhook \"inferenceservice.kserve-webhook-server.defaulter\": failed to call webhook: Post \"https://kserve-webhook-server-service.kserve.svc:443/mutate-serving-kserve-io-v1beta1-inferenceservice?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\""}]},"code":500}
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py:238: ApiException
During handling of the above exception, another exception occurred:
rest_v1_client =
@pytest.mark.graph
@pytest.mark.kourier
@pytest.mark.asyncio(scope="session")
async def test_ig_scenario4(rest_v1_client):
"""
Scenario: Switch graph with 1 step as hard dependency and other one as soft dependency.
Will be testing 3 cases in this test case:
Expectation:
Case 1. IG will return response of error_isvc when condition for that step matches
Case 2. IG will return response of success_isvc when condition for that step matches
Case 3. IG will return 404 with error message when no condition matches
{
"error": "Failed to process request",
"cause": "None of the routes matched with the switch condition",
}
"""
logger.info("Starting test test_ig_scenario4")
suffix = str(uuid.uuid4())[1:6]
success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test(
suffix
)
logger.info(f"success_isvc_name is {success_isvc_name}")
logger.info(f"error_isvc_name is {error_isvc_name}")
kserve_client = KServeClient(
config_file=os.environ.get("KUBECONFIG", "~/.kube/config")
)
> kserve_client.create(success_isvc)
graph/test_inference_graph.py:467:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
inferenceservice = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
namespace = 'kserve-ci-e2e-test', watch = False, timeout_seconds = 600
def create(
self, inferenceservice, namespace=None, watch=False, timeout_seconds=600
): # pylint:disable=inconsistent-return-statements
"""
Create the inference service
:param inferenceservice: inference service object
:param namespace: defaults to current or default namespace
:param watch: True to watch the created service until timeout elapsed or status is ready
:param timeout_seconds: timeout seconds for watch, default to 600s
:return: created inference service
"""
version = inferenceservice.api_version.split("/")[1]
if namespace is None:
namespace = utils.get_isvc_namespace(inferenceservice)
try:
outputs = self.api_instance.create_namespaced_custom_object(
constants.KSERVE_GROUP,
version,
namespace,
constants.KSERVE_PLURAL_INFERENCESERVICE,
inferenceservice,
)
except client.rest.ApiException as e:
> raise RuntimeError(
"Exception when calling CustomObjectsApi->create_namespaced_custom_object:\
%s\n"
% e
)
E RuntimeError: Exception when calling CustomObjectsApi->create_namespaced_custom_object: (500)
E Reason: Internal Server Error
E HTTP response headers: HTTPHeaderDict({'Audit-Id': 'a0ab5abb-05c2-4373-aa8f-488aec4a9794', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Strict-Transport-Security': 'max-age=31536000; includeSubDomains; preload', 'X-Kubernetes-Pf-Flowschema-Uid': '224291cc-bc8a-47e1-b358-99f64d47f314', 'X-Kubernetes-Pf-Prioritylevel-Uid': '53379352-efa0-40c7-b71a-ad76894c832e', 'Date': 'Thu, 16 Apr 2026 20:28:44 GMT', 'Content-Length': '769'})
E HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: failed calling webhook \"inferenceservice.kserve-webhook-server.defaulter\": failed to call webhook: Post \"https://kserve-webhook-server-service.kserve.svc:443/mutate-serving-kserve-io-v1beta1-inferenceservice?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","reason":"InternalError","details":{"causes":[{"message":"failed calling webhook \"inferenceservice.kserve-webhook-server.defaulter\": failed to call webhook: Post \"https://kserve-webhook-server-service.kserve.svc:443/mutate-serving-kserve-io-v1beta1-inferenceservice?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\""}]},"code":500}
../../python/kserve/kserve/api/kserve_client.py:153: RuntimeError
------------------------------ Captured log call -------------------------------
INFO kserve.trace:test_inference_graph.py:456 Starting test test_ig_scenario4
INFO kserve.trace:test_inference_graph.py:207 SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
INFO kserve.trace:test_inference_graph.py:208 ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
INFO kserve.trace:test_inference_graph.py:461 success_isvc_name is success-200-isvc-d7402
INFO kserve.trace:test_inference_graph.py:462 error_isvc_name is error-404-isvc-d7402
______________________________ test_ig_scenario2 _______________________________
[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python
@contextlib.contextmanager
def map_httpcore_exceptions() -> typing.Iterator[None]:
try:
> yield
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx/_transports/default.py:72:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
request =
async def handle_async_request(
self,
request: Request,
) -> Response:
assert isinstance(request.stream, AsyncByteStream)
req = httpcore.Request(
method=request.method,
url=httpcore.URL(
scheme=request.url.raw_scheme,
host=request.url.raw_host,
port=request.url.port,
target=request.url.raw_path,
),
headers=request.headers.raw,
content=request.stream,
extensions=request.extensions,
)
with map_httpcore_exceptions():
> resp = await self._pool.handle_async_request(req)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx/_transports/default.py:377:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
request =
async def handle_async_request(self, request: Request) -> Response:
"""
Send an HTTP request, and return an HTTP response.
This is the core implementation that is called into by `.request()` or `.stream()`.
"""
scheme = request.url.scheme.decode()
if scheme == "":
raise UnsupportedProtocol(
"Request URL is missing an 'http://' or 'https://' protocol."
)
if scheme not in ("http", "https", "ws", "wss"):
raise UnsupportedProtocol(
f"Request URL has an unsupported protocol '{scheme}://'."
)
timeouts = request.extensions.get("timeout", {})
timeout = timeouts.get("pool", None)
with self._optional_thread_lock:
# Add the incoming request to our request queue.
pool_request = AsyncPoolRequest(request)
self._requests.append(pool_request)
try:
while True:
with self._optional_thread_lock:
# Assign incoming requests to available connections,
# closing or creating new connections as required.
closing = self._assign_requests_to_connections()
await self._close_connections(closing)
# Wait until this request has an assigned connection.
connection = await pool_request.wait_for_connection(timeout=timeout)
try:
# Send the request on the assigned connection.
response = await connection.handle_async_request(
pool_request.request
)
except ConnectionNotAvailable:
# In some cases a connection may initially be available to
# handle a request, but then become unavailable.
#
# In this case we clear the connection and try again.
pool_request.clear_connection()
else:
break # pragma: nocover
except BaseException as exc:
with self._optional_thread_lock:
# For any exception or cancellation we remove the request from
# the queue, and then re-assign requests to connections.
self._requests.remove(pool_request)
closing = self._assign_requests_to_connections()
await self._close_connections(closing)
> raise exc from None
../../python/kserve/.venv/lib64/python3.11/site-packages/httpcore/_async/connection_pool.py:256:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
request =
async def handle_async_request(self, request: Request) -> Response:
"""
Send an HTTP request, and return an HTTP response.
This is the core implementation that is called into by `.request()` or `.stream()`.
"""
scheme = request.url.scheme.decode()
if scheme == "":
raise UnsupportedProtocol(
"Request URL is missing an 'http://' or 'https://' protocol."
)
if scheme not in ("http", "https", "ws", "wss"):
raise UnsupportedProtocol(
f"Request URL has an unsupported protocol '{scheme}://'."
)
timeouts = request.extensions.get("timeout", {})
timeout = timeouts.get("pool", None)
with self._optional_thread_lock:
# Add the incoming request to our request queue.
pool_request = AsyncPoolRequest(request)
self._requests.append(pool_request)
try:
while True:
with self._optional_thread_lock:
# Assign incoming requests to available connections,
# closing or creating new connections as required.
closing = self._assign_requests_to_connections()
await self._close_connections(closing)
# Wait until this request has an assigned connection.
connection = await pool_request.wait_for_connection(timeout=timeout)
try:
# Send the request on the assigned connection.
> response = await connection.handle_async_request(
pool_request.request
)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpcore/_async/connection_pool.py:236:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = , request =
async def handle_async_request(self, request: Request) -> Response:
if not self.can_handle_request(request.url.origin):
raise RuntimeError(
f"Attempted to send request to {request.url.origin} on connection to {self._origin}"
)
try:
async with self._request_lock:
if self._connection is None:
stream = await self._connect(request)
ssl_object = stream.get_extra_info("ssl_object")
http2_negotiated = (
ssl_object is not None
and ssl_object.selected_alpn_protocol() == "h2"
)
if http2_negotiated or (self._http2 and not self._http1):
from .http2 import AsyncHTTP2Connection
self._connection = AsyncHTTP2Connection(
origin=self._origin,
stream=stream,
keepalive_expiry=self._keepalive_expiry,
)
else:
self._connection = AsyncHTTP11Connection(
origin=self._origin,
stream=stream,
keepalive_expiry=self._keepalive_expiry,
)
except BaseException as exc:
self._connect_failed = True
> raise exc
../../python/kserve/.venv/lib64/python3.11/site-packages/httpcore/_async/connection.py:101:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = , request =
async def handle_async_request(self, request: Request) -> Response:
if not self.can_handle_request(request.url.origin):
raise RuntimeError(
f"Attempted to send request to {request.url.origin} on connection to {self._origin}"
)
try:
async with self._request_lock:
if self._connection is None:
> stream = await self._connect(request)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpcore/_async/connection.py:78:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = , request =
async def _connect(self, request: Request) -> AsyncNetworkStream:
timeouts = request.extensions.get("timeout", {})
sni_hostname = request.extensions.get("sni_hostname", None)
timeout = timeouts.get("connect", None)
retries_left = self._retries
delays = exponential_backoff(factor=RETRIES_BACKOFF_FACTOR)
while True:
try:
if self._uds is None:
kwargs = {
"host": self._origin.host.decode("ascii"),
"port": self._origin.port,
"local_address": self._local_address,
"timeout": timeout,
"socket_options": self._socket_options,
}
async with Trace("connect_tcp", logger, request, kwargs) as trace:
> stream = await self._network_backend.connect_tcp(**kwargs)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpcore/_async/connection.py:124:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
host = 'sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
port = 443, timeout = 180, local_address = None, socket_options = None
async def connect_tcp(
self,
host: str,
port: int,
timeout: float | None = None,
local_address: str | None = None,
socket_options: typing.Iterable[SOCKET_OPTION] | None = None,
) -> AsyncNetworkStream:
await self._init_backend()
> return await self._backend.connect_tcp(
host,
port,
timeout=timeout,
local_address=local_address,
socket_options=socket_options,
)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpcore/_backends/auto.py:31:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
host = 'sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
port = 443, timeout = 180, local_address = None, socket_options = []
async def connect_tcp(
self,
host: str,
port: int,
timeout: float | None = None,
local_address: str | None = None,
socket_options: typing.Iterable[SOCKET_OPTION] | None = None,
) -> AsyncNetworkStream: # pragma: nocover
if socket_options is None:
socket_options = []
exc_map = {
TimeoutError: ConnectTimeout,
OSError: ConnectError,
anyio.BrokenResourceError: ConnectError,
}
> with map_exceptions(exc_map):
../../python/kserve/.venv/lib64/python3.11/site-packages/httpcore/_backends/anyio.py:113:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
typ =
value = gaierror(-2, 'Name or service not known')
traceback =
def __exit__(self, typ, value, traceback):
if typ is None:
try:
next(self.gen)
except StopIteration:
return False
else:
try:
raise RuntimeError("generator didn't stop")
finally:
self.gen.close()
else:
if value is None:
# Need to force instantiation so we can reliably
# tell if we get the same exception back
value = typ()
try:
> self.gen.throw(typ, value, traceback)
/usr/lib64/python3.11/contextlib.py:158:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
map = {: , : , : }
@contextlib.contextmanager
def map_exceptions(map: ExceptionMapping) -> typing.Iterator[None]:
try:
yield
except Exception as exc: # noqa: PIE786
for from_exc, to_exc in map.items():
if isinstance(exc, from_exc):
> raise to_exc(exc) from exc
E httpcore.ConnectError: [Errno -2] Name or service not known
../../python/kserve/.venv/lib64/python3.11/site-packages/httpcore/_exceptions.py:14: ConnectError
The above exception was the direct cause of the following exception:
rest_v1_client =
@pytest.mark.graph
@pytest.mark.kourier
@pytest.mark.asyncio(scope="session")
async def test_ig_scenario2(rest_v1_client):
"""
Scenario: Sequence graph with 2 steps that are both soft dependencies.
error_isvc (soft) -> success_isvc(soft)
Expectation: IG will return response of success_isvc and predict_ig will not raise any exception
:return:
"""
logger.info("Starting test test_ig_scenario2")
suffix = str(uuid.uuid4())[1:6]
success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test(
suffix
)
logger.info(f"success_isvc_name is {success_isvc_name}")
logger.info(f"error_isvc_name is {error_isvc_name}")
# Create graph
graph_name = "-".join(["sequence-graph", suffix])
nodes = {
"root": V1alpha1InferenceRouter(
router_type="Sequence",
steps=[
V1alpha1InferenceStep(
service_name=error_isvc_name,
),
V1alpha1InferenceStep(
service_name=success_isvc_name,
data="$request",
),
],
)
}
graph_spec = V1alpha1InferenceGraphSpec(
nodes=nodes,
)
ig = V1alpha1InferenceGraph(
api_version=constants.KSERVE_V1ALPHA1,
kind=constants.KSERVE_KIND_INFERENCEGRAPH,
metadata=client.V1ObjectMeta(name=graph_name, namespace=KSERVE_TEST_NAMESPACE),
spec=graph_spec,
)
kserve_client = KServeClient(
config_file=os.environ.get("KUBECONFIG", "~/.kube/config")
)
kserve_client.create(success_isvc)
kserve_client.create(error_isvc)
kserve_client.wait_isvc_ready(success_isvc_name, namespace=KSERVE_TEST_NAMESPACE)
kserve_client.wait_isvc_ready(error_isvc_name, namespace=KSERVE_TEST_NAMESPACE)
kserve_client.create_inference_graph(ig)
kserve_client.wait_ig_ready(graph_name, namespace=KSERVE_TEST_NAMESPACE)
> response = await predict_ig(
rest_v1_client,
graph_name,
os.path.join(IG_TEST_RESOURCES_BASE_LOCATION, "custom_predictor_input.json"),
)
graph/test_inference_graph.py:360:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
client =
ig_name = 'sequence-graph-b1781'
input_path = 'graph/test-resources/custom_predictor_input.json'
version = 'v1alpha1', network_layer = 'istio'
async def predict_ig(
client: InferenceRESTClient,
ig_name,
input_path,
version=constants.KSERVE_V1ALPHA1_VERSION,
network_layer: str = "istio",
) -> Union[InferResponse, Dict]:
kserve_client = KServeClient(
config_file=os.environ.get("KUBECONFIG", "~/.kube/config")
)
ig = kserve_client.get_inference_graph(
ig_name,
namespace=KSERVE_TEST_NAMESPACE,
version=version,
)
scheme, cluster_ip, host, _ = get_isvc_endpoint(ig, network_layer)
url = f"{scheme}://{cluster_ip}"
> return await predict(client, url, host, input_path, is_graph=True)
common/utils.py:193:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
client =
url = 'https://sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
host = 'sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
input = 'graph/test-resources/custom_predictor_input.json', model_name = None
is_batch = False, is_graph = True
async def predict(
client: InferenceRESTClient,
url,
host,
input: Union[str, InferRequest],
model_name=None,
is_batch=False,
is_graph=False,
) -> Union[InferResponse, Dict, List[Union[Dict, InferResponse]]]:
if isinstance(input, str):
with open(input) as json_file:
data = json.load(json_file)
else:
data = input
headers = {"Host": host, "Content-Type": "application/json"}
if is_batch:
with futures.ThreadPoolExecutor(max_workers=4) as executor:
loop = asyncio.get_running_loop()
future_list = [
await loop.run_in_executor(
executor,
_predict,
client,
url,
input_data,
model_name,
headers,
is_graph,
)
for input_data in data
]
result = await asyncio.gather(*future_list)
else:
> result = await _predict(
client,
url,
data,
model_name=model_name,
headers=headers,
is_graph=is_graph,
)
common/utils.py:120:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
client =
url = 'https://sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
input_data = {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
model_name = None
headers = {'Content-Type': 'application/json', 'Host': 'sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd...c922f58bfea.prod.konfluxeaas.com', 'content-type': 'application/octet-stream', 'inference-header-content-length': '51'}
is_graph = True
async def _predict(
client: InferenceRESTClient,
url,
input_data,
model_name,
headers=None,
is_graph=False,
*,
retries: int = 5,
retry_delay: int = 5,
) -> Union[InferResponse, Dict]:
transient_status_codes = {502, 503, 504}
logger.info("Sending Header = %s", headers)
logger.info("base url = %s", url)
for attempt in range(retries):
try:
> return await client.infer(
url,
input_data,
model_name=model_name,
headers=headers,
is_graph_endpoint=is_graph,
)
common/utils.py:148:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
base_url = 'https://sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
data = b'{"instances":[[6.8,2.8,4.8,1.4],[6.0,3.4,4.5,1.6]]}', model_name = None
headers = {'Content-Type': 'application/json', 'Host': 'sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd...c922f58bfea.prod.konfluxeaas.com', 'content-type': 'application/octet-stream', 'inference-header-content-length': '51'}
response_headers = None, is_graph_endpoint = True
timeout =
async def infer(
self,
base_url: Union[httpx.URL, str],
data: Union[InferRequest, dict],
model_name: Optional[str] = None,
headers: Optional[Mapping[str, str]] = None,
response_headers: Dict[str, str] = None,
is_graph_endpoint: bool = False,
timeout: Union[float, None, tuple, httpx.Timeout] = httpx.USE_CLIENT_DEFAULT,
) -> Union[InferResponse, Dict]:
"""
Run asynchronous inference using the supplied data.
:param base_url: Base url of the inference server. E.g. https://example.com:443, https://example.com:443/serving
:param data: Input data as InferRequest object.
:param model_name: (optional) Name of the model as string. If is_graph_endpoint is true this can be omitted.
If is_graph_endpoint is False and model_name is None, this will raise ValueError.
:param headers: (optional) HTTP headers to include when sending request.
:param is_graph_endpoint: (optional) If set to True the base_url will be considered as an inference graph
endpoint and will be used as it is for making the request regardless of the
protocol specified in the RESTConfig. The result will be returned as dict object.
Defaults to False.
:param timeout: (optional) The maximum end-to-end time, in seconds, the request is allowed to take. This will
override the timeout in the RESTConfig. The default value is 60 seconds.
To disable timeout explicitly set it to 'None'.
:return: Inference result as InferResponse object or python dict.
:raises HTTPStatusError for response codes other than 2xx.
:raises UnsupportedProtocol if the specified protocol version is not supported.
"""
if is_graph_endpoint:
url = base_url
elif model_name is None:
raise ValueError("model_name should not be 'None'")
elif is_v1(self._config.protocol):
url = self._construct_url(
base_url, f"{self._config.protocol}/models/{model_name}:predict"
)
elif is_v2(self._config.protocol):
url = self._construct_url(
base_url, f"{self._config.protocol}/models/{model_name}/infer"
)
else:
raise UnsupportedProtocol(self._config.protocol)
if self._config.verbose:
logger.info("url: %s", url)
logger.info("request data: %s", data)
if isinstance(data, InferRequest):
data, json_length = data.to_rest()
if json_length:
headers = headers or {}
headers[INFERENCE_CONTENT_LENGTH_HEADER] = str(json_length)
headers["content-type"] = "application/octet-stream"
if isinstance(data, dict):
data = orjson.dumps(data)
headers = headers or {}
headers.setdefault(INFERENCE_CONTENT_LENGTH_HEADER, str(len(data)))
headers.setdefault("content-type", "application/octet-stream")
> response = await self._client.post(
url, content=data, headers=headers, timeout=timeout
)
../../python/kserve/kserve/inference_client.py:503:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
url = 'https://sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
async def post(
self,
url: URL | str,
*,
content: RequestContent | None = None,
data: RequestData | None = None,
files: RequestFiles | None = None,
json: typing.Any | None = None,
params: QueryParamTypes | None = None,
headers: HeaderTypes | None = None,
cookies: CookieTypes | None = None,
auth: AuthTypes | UseClientDefault = USE_CLIENT_DEFAULT,
follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT,
timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT,
extensions: RequestExtensions | None = None,
) -> Response:
"""
Send a `POST` request.
**Parameters**: See `httpx.request`.
"""
> return await self.request(
"POST",
url,
content=content,
data=data,
files=files,
json=json,
params=params,
headers=headers,
cookies=cookies,
auth=auth,
follow_redirects=follow_redirects,
timeout=timeout,
extensions=extensions,
)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx/_client.py:1905:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = , method = 'POST'
url = 'https://sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com'
async def request(
self,
method: str,
url: URL | str,
*,
content: RequestContent | None = None,
data: RequestData | None = None,
files: RequestFiles | None = None,
json: typing.Any | None = None,
params: QueryParamTypes | None = None,
headers: HeaderTypes | None = None,
cookies: CookieTypes | None = None,
auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT,
follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT,
timeout: TimeoutTypes | UseClientDefault = USE_CLIENT_DEFAULT,
extensions: RequestExtensions | None = None,
) -> Response:
"""
Build and send a request.
Equivalent to:
```python
request = client.build_request(...)
response = await client.send(request, ...)
```
See `AsyncClient.build_request()`, `AsyncClient.send()`
and [Merging of configuration][0] for how the various parameters
are merged with client-level configuration.
[0]: /advanced/clients/#merging-of-configuration
"""
if cookies is not None: # pragma: no cover
message = (
"Setting per-request cookies=<...> is being deprecated, because "
"the expected behaviour on cookie persistence is ambiguous. Set "
"cookies directly on the client instance instead."
)
warnings.warn(message, DeprecationWarning)
request = self.build_request(
method=method,
url=url,
content=content,
data=data,
files=files,
json=json,
params=params,
headers=headers,
cookies=cookies,
timeout=timeout,
extensions=extensions,
)
> return await self.send(request, auth=auth, follow_redirects=follow_redirects)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx/_client.py:1585:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
request =
async def send(
self,
request: Request,
*,
stream: bool = False,
auth: AuthTypes | UseClientDefault | None = USE_CLIENT_DEFAULT,
follow_redirects: bool | UseClientDefault = USE_CLIENT_DEFAULT,
) -> Response:
"""
Send a request.
The request is sent as-is, unmodified.
Typically you'll want to build one with `AsyncClient.build_request()`
so that any client-level configuration is merged into the request,
but passing an explicit `httpx.Request()` is supported as well.
See also: [Request instances][0]
[0]: /advanced/clients/#request-instances
"""
if self._state == ClientState.CLOSED:
raise RuntimeError("Cannot send a request, as the client has been closed.")
self._state = ClientState.OPENED
follow_redirects = (
self.follow_redirects
if isinstance(follow_redirects, UseClientDefault)
else follow_redirects
)
self._set_timeout(request)
auth = self._build_request_auth(request, auth)
> response = await self._send_handling_auth(
request,
auth=auth,
follow_redirects=follow_redirects,
history=[],
)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx/_client.py:1674:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
request =
auth = , follow_redirects = False
history = []
async def _send_handling_auth(
self,
request: Request,
auth: Auth,
follow_redirects: bool,
history: list[Response],
) -> Response:
auth_flow = auth.async_auth_flow(request)
try:
request = await auth_flow.__anext__()
while True:
> response = await self._send_handling_redirects(
request,
follow_redirects=follow_redirects,
history=history,
)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx/_client.py:1702:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
request =
follow_redirects = False, history = []
async def _send_handling_redirects(
self,
request: Request,
follow_redirects: bool,
history: list[Response],
) -> Response:
while True:
if len(history) > self.max_redirects:
raise TooManyRedirects(
"Exceeded maximum allowed redirects.", request=request
)
for hook in self._event_hooks["request"]:
await hook(request)
> response = await self._send_single_request(request)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx/_client.py:1739:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
request =
async def _send_single_request(self, request: Request) -> Response:
"""
Sends a single request, without handling any redirections.
"""
transport = self._transport_for_url(request.url)
timer = Timer()
await timer.async_start()
if not isinstance(request.stream, AsyncByteStream):
raise RuntimeError(
"Attempted to send an sync request with an AsyncClient instance."
)
with request_context(request=request):
> response = await transport.handle_async_request(request)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx/_client.py:1776:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
request =
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
"""Sends an HTTP request, possibly with retries.
Args:
request: The request to perform.
Returns:
The final response.
"""
if self._async_transport is None:
raise RuntimeError("Async request received but no async transport available")
logger.debug("handle_async_request started request=%s", request)
if self.retry.is_retryable_method(request.method):
send_method = partial(self._async_transport.handle_async_request)
> response = await self._retry_operation_async(request, send_method)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx_retries/transport.py:116:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
request =
send_method = functools.partial(>)
async def _retry_operation_async(
self,
request: httpx.Request,
send_method: Callable[..., Coroutine[Any, Any, httpx.Response]],
) -> httpx.Response:
retry = self.retry
response: Union[httpx.Response, Exception, None] = None
while True:
if response is not None:
if isinstance(response, httpx.Response):
await response.aclose()
logger.debug(
"_retry_operation_async retrying request=%s response=%s retry=%s", request, response, retry
)
retry = retry.increment()
await retry.asleep(response)
try:
> response = await send_method(request)
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx_retries/transport.py:171:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
request =
async def handle_async_request(
self,
request: Request,
) -> Response:
assert isinstance(request.stream, AsyncByteStream)
req = httpcore.Request(
method=request.method,
url=httpcore.URL(
scheme=request.url.raw_scheme,
host=request.url.raw_host,
port=request.url.port,
target=request.url.raw_path,
),
headers=request.headers.raw,
content=request.stream,
extensions=request.extensions,
)
> with map_httpcore_exceptions():
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx/_transports/default.py:376:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
typ =
value = ConnectError(gaierror(-2, 'Name or service not known'))
traceback =
def __exit__(self, typ, value, traceback):
if typ is None:
try:
next(self.gen)
except StopIteration:
return False
else:
try:
raise RuntimeError("generator didn't stop")
finally:
self.gen.close()
else:
if value is None:
# Need to force instantiation so we can reliably
# tell if we get the same exception back
value = typ()
try:
> self.gen.throw(typ, value, traceback)
/usr/lib64/python3.11/contextlib.py:158:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
@contextlib.contextmanager
def map_httpcore_exceptions() -> typing.Iterator[None]:
try:
yield
except Exception as exc:
mapped_exc = None
for from_exc, to_exc in HTTPCORE_EXC_MAP.items():
if not isinstance(exc, from_exc):
continue
# We want to map to the most specific exception we can find.
# Eg if `exc` is an `httpcore.ReadTimeout`, we want to map to
# `httpx.ReadTimeout`, not just `httpx.TimeoutException`.
if mapped_exc is None or issubclass(to_exc, mapped_exc):
mapped_exc = to_exc
if mapped_exc is None: # pragma: no cover
raise
message = str(exc)
> raise mapped_exc(message) from exc
E httpx.ConnectError: [Errno -2] Name or service not known
../../python/kserve/.venv/lib64/python3.11/site-packages/httpx/_transports/default.py:89: ConnectError
------------------------------ Captured log call -------------------------------
INFO kserve.trace:test_inference_graph.py:314 Starting test test_ig_scenario2
INFO kserve.trace:test_inference_graph.py:207 SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
INFO kserve.trace:test_inference_graph.py:208 ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
INFO kserve.trace:test_inference_graph.py:319 success_isvc_name is success-200-isvc-b1781
INFO kserve.trace:test_inference_graph.py:320 error_isvc_name is error-404-isvc-b1781
INFO kserve.trace:utils.py:335 CI_USE_ISVC_HOST = 1
INFO kserve.trace:utils.py:336 Host from isvc status URL = sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:utils.py:337 Network layer = istio
INFO kserve.trace:utils.py:340 Using external route host: sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:utils.py:144 Sending Header = {'Host': 'sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com', 'Content-Type': 'application/json'}
INFO kserve.trace:utils.py:145 base url = https://sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:490 url: https://sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
INFO httpx:_client.py:1786 HTTP Request: POST https://sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com "HTTP/1.0 503 Service Unavailable"
INFO kserve.trace:inference_client.py:507 response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:166 Transient error on attempt 1/5, retrying in 5s...
INFO kserve.trace:inference_client.py:490 url: https://sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
INFO httpx:_client.py:1786 HTTP Request: POST https://sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com "HTTP/1.0 503 Service Unavailable"
INFO kserve.trace:inference_client.py:507 response code: 503, content:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:156 HTTP 503 response body:
Application is not available
The application is currently not serving requests at this endpoint. It may not have been started or is still starting.
Possible reasons you are seeing this page:
-
The host doesn't exist.
Make sure the hostname was typed correctly and that a route matching this hostname exists.
-
The host exists, but doesn't have a matching path.
Check if the URL path was typed correctly and that the route was created using the desired path.
-
Route and path matches, but all pods are down.
Make sure that the resources exposed by this route (pods, services, deployment configs, etc) have at least one pod running.
INFO kserve.trace:utils.py:166 Transient error on attempt 2/5, retrying in 5s...
INFO kserve.trace:inference_client.py:490 url: https://sequence-graph-b1781-route-kserve-ci-e2e-test.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com
INFO kserve.trace:inference_client.py:491 request data: {'instances': [[6.8, 2.8, 4.8, 1.4], [6.0, 3.4, 4.5, 1.6]]}
______________________________ test_ig_scenario3 _______________________________
[gw0] linux -- Python 3.11.13 /workspace/source/python/kserve/.venv/bin/python
self =
inferenceservice = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
namespace = 'kserve-ci-e2e-test', watch = False, timeout_seconds = 600
def create(
self, inferenceservice, namespace=None, watch=False, timeout_seconds=600
): # pylint:disable=inconsistent-return-statements
"""
Create the inference service
:param inferenceservice: inference service object
:param namespace: defaults to current or default namespace
:param watch: True to watch the created service until timeout elapsed or status is ready
:param timeout_seconds: timeout seconds for watch, default to 600s
:return: created inference service
"""
version = inferenceservice.api_version.split("/")[1]
if namespace is None:
namespace = utils.get_isvc_namespace(inferenceservice)
try:
> outputs = self.api_instance.create_namespaced_custom_object(
constants.KSERVE_GROUP,
version,
namespace,
constants.KSERVE_PLURAL_INFERENCESERVICE,
inferenceservice,
)
../../python/kserve/kserve/api/kserve_client.py:145:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
group = 'serving.kserve.io', version = 'v1beta1'
namespace = 'kserve-ci-e2e-test', plural = 'inferenceservices'
body = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
kwargs = {'_return_http_data_only': True}
def create_namespaced_custom_object(self, group, version, namespace, plural, body, **kwargs): # noqa: E501
"""create_namespaced_custom_object # noqa: E501
Creates a namespace scoped Custom object # noqa: E501
This method makes a synchronous HTTP request by default. To make an
asynchronous HTTP request, please pass async_req=True
>>> thread = api.create_namespaced_custom_object(group, version, namespace, plural, body, async_req=True)
>>> result = thread.get()
:param async_req bool: execute request asynchronously
:param str group: The custom resource's group name (required)
:param str version: The custom resource's version (required)
:param str namespace: The custom resource's namespace (required)
:param str plural: The custom resource's plural name. For TPRs this would be lowercase plural kind. (required)
:param object body: The JSON schema of the Resource to create. (required)
:param str pretty: If 'true', then the output is pretty printed.
:param str dry_run: When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed
:param str field_manager: fieldManager is a name associated with the actor or entity that is making these changes. The value must be less than or 128 characters long, and only contain printable characters, as defined by https://golang.org/pkg/unicode/#IsPrint.
:param str field_validation: fieldValidation instructs the server on how to handle objects in the request (POST/PUT/PATCH) containing unknown or duplicate fields. Valid values are: - Ignore: This will ignore any unknown fields that are silently dropped from the object, and will ignore all but the last duplicate field that the decoder encounters. This is the default behavior prior to v1.23. - Warn: This will send a warning via the standard warning response header for each unknown field that is dropped from the object, and for each duplicate field that is encountered. The request will still succeed if there are no other errors, and will only persist the last of any duplicate fields. This is the default in v1.23+ - Strict: This will fail the request with a BadRequest error if any unknown fields would be dropped from the object, or if any duplicate fields are present. The error returned from the server will contain all unknown and duplicate fields encountered. (optional)
:param _preload_content: if False, the urllib3.HTTPResponse object will
be returned without reading/decoding response
data. Default is True.
:param _request_timeout: timeout setting for this request. If one
number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
:return: object
If the method is called asynchronously,
returns the request thread.
"""
kwargs['_return_http_data_only'] = True
> return self.create_namespaced_custom_object_with_http_info(group, version, namespace, plural, body, **kwargs) # noqa: E501
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/custom_objects_api.py:231:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
group = 'serving.kserve.io', version = 'v1beta1'
namespace = 'kserve-ci-e2e-test', plural = 'inferenceservices'
body = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
kwargs = {'_return_http_data_only': True}
local_var_params = {'_return_http_data_only': True, 'all_params': ['group', 'version', 'namespace', 'plural', 'body', 'pretty', ...], 'au... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}, ...}
all_params = ['group', 'version', 'namespace', 'plural', 'body', 'pretty', ...]
key = '_return_http_data_only', val = True, collection_formats = {}
path_params = {'group': 'serving.kserve.io', 'namespace': 'kserve-ci-e2e-test', 'plural': 'inferenceservices', 'version': 'v1beta1'}
query_params = []
def create_namespaced_custom_object_with_http_info(self, group, version, namespace, plural, body, **kwargs): # noqa: E501
"""create_namespaced_custom_object # noqa: E501
Creates a namespace scoped Custom object # noqa: E501
This method makes a synchronous HTTP request by default. To make an
asynchronous HTTP request, please pass async_req=True
>>> thread = api.create_namespaced_custom_object_with_http_info(group, version, namespace, plural, body, async_req=True)
>>> result = thread.get()
:param async_req bool: execute request asynchronously
:param str group: The custom resource's group name (required)
:param str version: The custom resource's version (required)
:param str namespace: The custom resource's namespace (required)
:param str plural: The custom resource's plural name. For TPRs this would be lowercase plural kind. (required)
:param object body: The JSON schema of the Resource to create. (required)
:param str pretty: If 'true', then the output is pretty printed.
:param str dry_run: When present, indicates that modifications should not be persisted. An invalid or unrecognized dryRun directive will result in an error response and no further processing of the request. Valid values are: - All: all dry run stages will be processed
:param str field_manager: fieldManager is a name associated with the actor or entity that is making these changes. The value must be less than or 128 characters long, and only contain printable characters, as defined by https://golang.org/pkg/unicode/#IsPrint.
:param str field_validation: fieldValidation instructs the server on how to handle objects in the request (POST/PUT/PATCH) containing unknown or duplicate fields. Valid values are: - Ignore: This will ignore any unknown fields that are silently dropped from the object, and will ignore all but the last duplicate field that the decoder encounters. This is the default behavior prior to v1.23. - Warn: This will send a warning via the standard warning response header for each unknown field that is dropped from the object, and for each duplicate field that is encountered. The request will still succeed if there are no other errors, and will only persist the last of any duplicate fields. This is the default in v1.23+ - Strict: This will fail the request with a BadRequest error if any unknown fields would be dropped from the object, or if any duplicate fields are present. The error returned from the server will contain all unknown and duplicate fields encountered. (optional)
:param _return_http_data_only: response data without head status code
and headers
:param _preload_content: if False, the urllib3.HTTPResponse object will
be returned without reading/decoding response
data. Default is True.
:param _request_timeout: timeout setting for this request. If one
number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
:return: tuple(object, status_code(int), headers(HTTPHeaderDict))
If the method is called asynchronously,
returns the request thread.
"""
local_var_params = locals()
all_params = [
'group',
'version',
'namespace',
'plural',
'body',
'pretty',
'dry_run',
'field_manager',
'field_validation'
]
all_params.extend(
[
'async_req',
'_return_http_data_only',
'_preload_content',
'_request_timeout'
]
)
for key, val in six.iteritems(local_var_params['kwargs']):
if key not in all_params:
raise ApiTypeError(
"Got an unexpected keyword argument '%s'"
" to method create_namespaced_custom_object" % key
)
local_var_params[key] = val
del local_var_params['kwargs']
# verify the required parameter 'group' is set
if self.api_client.client_side_validation and ('group' not in local_var_params or # noqa: E501
local_var_params['group'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `group` when calling `create_namespaced_custom_object`") # noqa: E501
# verify the required parameter 'version' is set
if self.api_client.client_side_validation and ('version' not in local_var_params or # noqa: E501
local_var_params['version'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `version` when calling `create_namespaced_custom_object`") # noqa: E501
# verify the required parameter 'namespace' is set
if self.api_client.client_side_validation and ('namespace' not in local_var_params or # noqa: E501
local_var_params['namespace'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `namespace` when calling `create_namespaced_custom_object`") # noqa: E501
# verify the required parameter 'plural' is set
if self.api_client.client_side_validation and ('plural' not in local_var_params or # noqa: E501
local_var_params['plural'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `plural` when calling `create_namespaced_custom_object`") # noqa: E501
# verify the required parameter 'body' is set
if self.api_client.client_side_validation and ('body' not in local_var_params or # noqa: E501
local_var_params['body'] is None): # noqa: E501
raise ApiValueError("Missing the required parameter `body` when calling `create_namespaced_custom_object`") # noqa: E501
collection_formats = {}
path_params = {}
if 'group' in local_var_params:
path_params['group'] = local_var_params['group'] # noqa: E501
if 'version' in local_var_params:
path_params['version'] = local_var_params['version'] # noqa: E501
if 'namespace' in local_var_params:
path_params['namespace'] = local_var_params['namespace'] # noqa: E501
if 'plural' in local_var_params:
path_params['plural'] = local_var_params['plural'] # noqa: E501
query_params = []
if 'pretty' in local_var_params and local_var_params['pretty'] is not None: # noqa: E501
query_params.append(('pretty', local_var_params['pretty'])) # noqa: E501
if 'dry_run' in local_var_params and local_var_params['dry_run'] is not None: # noqa: E501
query_params.append(('dryRun', local_var_params['dry_run'])) # noqa: E501
if 'field_manager' in local_var_params and local_var_params['field_manager'] is not None: # noqa: E501
query_params.append(('fieldManager', local_var_params['field_manager'])) # noqa: E501
if 'field_validation' in local_var_params and local_var_params['field_validation'] is not None: # noqa: E501
query_params.append(('fieldValidation', local_var_params['field_validation'])) # noqa: E501
header_params = {}
form_params = []
local_var_files = {}
body_params = None
if 'body' in local_var_params:
body_params = local_var_params['body']
# HTTP header `Accept`
header_params['Accept'] = self.api_client.select_header_accept(
['application/json']) # noqa: E501
# Authentication setting
auth_settings = ['BearerToken'] # noqa: E501
> return self.api_client.call_api(
'/apis/{group}/{version}/namespaces/{namespace}/{plural}', 'POST',
path_params,
query_params,
header_params,
body=body_params,
post_params=form_params,
files=local_var_files,
response_type='object', # noqa: E501
auth_settings=auth_settings,
async_req=local_var_params.get('async_req'),
_return_http_data_only=local_var_params.get('_return_http_data_only'), # noqa: E501
_preload_content=local_var_params.get('_preload_content', True),
_request_timeout=local_var_params.get('_request_timeout'),
collection_formats=collection_formats)
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api/custom_objects_api.py:354:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
resource_path = '/apis/{group}/{version}/namespaces/{namespace}/{plural}'
method = 'POST'
path_params = {'group': 'serving.kserve.io', 'namespace': 'kserve-ci-e2e-test', 'plural': 'inferenceservices', 'version': 'v1beta1'}
query_params = []
header_params = {'Accept': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
body = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
post_params = [], files = {}, response_type = 'object'
auth_settings = ['BearerToken'], async_req = None, _return_http_data_only = True
collection_formats = {}, _preload_content = True, _request_timeout = None
_host = None
def call_api(self, resource_path, method,
path_params=None, query_params=None, header_params=None,
body=None, post_params=None, files=None,
response_type=None, auth_settings=None, async_req=None,
_return_http_data_only=None, collection_formats=None,
_preload_content=True, _request_timeout=None, _host=None):
"""Makes the HTTP request (synchronous) and returns deserialized data.
To make an async_req request, set the async_req parameter.
:param resource_path: Path to method endpoint.
:param method: Method to call.
:param path_params: Path parameters in the url.
:param query_params: Query parameters in the url.
:param header_params: Header parameters to be
placed in the request header.
:param body: Request body.
:param post_params dict: Request post form parameters,
for `application/x-www-form-urlencoded`, `multipart/form-data`.
:param auth_settings list: Auth Settings names for the request.
:param response: Response data type.
:param files dict: key -> filename, value -> filepath,
for `multipart/form-data`.
:param async_req bool: execute request asynchronously
:param _return_http_data_only: response data without head status code
and headers
:param collection_formats: dict of collection formats for path, query,
header, and post parameters.
:param _preload_content: if False, the urllib3.HTTPResponse object will
be returned without reading/decoding response
data. Default is True.
:param _request_timeout: timeout setting for this request. If one
number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
:return:
If async_req parameter is True,
the request will be called asynchronously.
The method will return the request thread.
If parameter async_req is False or missing,
then the method will return the response directly.
"""
if not async_req:
> return self.__call_api(resource_path, method,
path_params, query_params, header_params,
body, post_params, files,
response_type, auth_settings,
_return_http_data_only, collection_formats,
_preload_content, _request_timeout, _host)
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:348:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
resource_path = '/apis/serving.kserve.io/v1beta1/namespaces/kserve-ci-e2e-test/inferenceservices'
method = 'POST'
path_params = [('group', 'serving.kserve.io'), ('version', 'v1beta1'), ('namespace', 'kserve-ci-e2e-test'), ('plural', 'inferenceservices')]
query_params = []
header_params = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
body = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'name': 'success-200-isvc-aeddd',...tahub/success-200-isvc:odh-pr-1326', 'name': 'kserve-container', 'resources': {'limits': {...}, 'requests': {...}}}]}}}
post_params = [], files = {}, response_type = 'object'
auth_settings = ['BearerToken'], _return_http_data_only = True
collection_formats = {}, _preload_content = True, _request_timeout = None
_host = None
def __call_api(
self, resource_path, method, path_params=None,
query_params=None, header_params=None, body=None, post_params=None,
files=None, response_type=None, auth_settings=None,
_return_http_data_only=None, collection_formats=None,
_preload_content=True, _request_timeout=None, _host=None):
config = self.configuration
# header parameters
header_params = header_params or {}
header_params.update(self.default_headers)
if self.cookie:
header_params['Cookie'] = self.cookie
if header_params:
header_params = self.sanitize_for_serialization(header_params)
header_params = dict(self.parameters_to_tuples(header_params,
collection_formats))
# path parameters
if path_params:
path_params = self.sanitize_for_serialization(path_params)
path_params = self.parameters_to_tuples(path_params,
collection_formats)
for k, v in path_params:
# specified safe chars, encode everything
resource_path = resource_path.replace(
'{%s}' % k,
quote(str(v), safe=config.safe_chars_for_path_param)
)
# query parameters
if query_params:
query_params = self.sanitize_for_serialization(query_params)
query_params = self.parameters_to_tuples(query_params,
collection_formats)
# post parameters
if post_params or files:
post_params = post_params if post_params else []
post_params = self.sanitize_for_serialization(post_params)
post_params = self.parameters_to_tuples(post_params,
collection_formats)
post_params.extend(self.files_parameters(files))
# auth setting
self.update_params_for_auth(header_params, query_params, auth_settings)
# body
if body:
body = self.sanitize_for_serialization(body)
# request url
if _host is None:
url = self.configuration.host + resource_path
else:
# use server/host defined in path or operation instead
url = _host + resource_path
# perform request and return response
> response_data = self.request(
method, url, query_params=query_params, headers=header_params,
post_params=post_params, body=body,
_preload_content=_preload_content,
_request_timeout=_request_timeout)
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:180:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
method = 'POST'
url = 'https://a9b56e18e9e4c45dcaeda4bb15f69bb8-9f16fbcdf636ba83.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1beta1/namespaces/kserve-ci-e2e-test/inferenceservices'
query_params = []
headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
post_params = []
body = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'name': 'success-200-isvc-aeddd',...tahub/success-200-isvc:odh-pr-1326', 'name': 'kserve-container', 'resources': {'limits': {...}, 'requests': {...}}}]}}}
_preload_content = True, _request_timeout = None
def request(self, method, url, query_params=None, headers=None,
post_params=None, body=None, _preload_content=True,
_request_timeout=None):
"""Makes the HTTP request using RESTClient."""
if method == "GET":
return self.rest_client.GET(url,
query_params=query_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
headers=headers)
elif method == "HEAD":
return self.rest_client.HEAD(url,
query_params=query_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
headers=headers)
elif method == "OPTIONS":
return self.rest_client.OPTIONS(url,
query_params=query_params,
headers=headers,
_preload_content=_preload_content,
_request_timeout=_request_timeout)
elif method == "POST":
> return self.rest_client.POST(url,
query_params=query_params,
headers=headers,
post_params=post_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/api_client.py:391:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
url = 'https://a9b56e18e9e4c45dcaeda4bb15f69bb8-9f16fbcdf636ba83.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1beta1/namespaces/kserve-ci-e2e-test/inferenceservices'
headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
query_params = [], post_params = []
body = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'name': 'success-200-isvc-aeddd',...tahub/success-200-isvc:odh-pr-1326', 'name': 'kserve-container', 'resources': {'limits': {...}, 'requests': {...}}}]}}}
_preload_content = True, _request_timeout = None
def POST(self, url, headers=None, query_params=None, post_params=None,
body=None, _preload_content=True, _request_timeout=None):
> return self.request("POST", url,
headers=headers,
query_params=query_params,
post_params=post_params,
_preload_content=_preload_content,
_request_timeout=_request_timeout,
body=body)
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py:279:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
method = 'POST'
url = 'https://a9b56e18e9e4c45dcaeda4bb15f69bb8-9f16fbcdf636ba83.elb.us-east-1.amazonaws.com:6443/apis/serving.kserve.io/v1beta1/namespaces/kserve-ci-e2e-test/inferenceservices'
query_params = []
headers = {'Accept': 'application/json', 'Content-Type': 'application/json', 'User-Agent': 'OpenAPI-Generator/32.0.1/python'}
body = {'apiVersion': 'serving.kserve.io/v1beta1', 'kind': 'InferenceService', 'metadata': {'name': 'success-200-isvc-aeddd',...tahub/success-200-isvc:odh-pr-1326', 'name': 'kserve-container', 'resources': {'limits': {...}, 'requests': {...}}}]}}}
post_params = {}, _preload_content = True, _request_timeout = None
def request(self, method, url, query_params=None, headers=None,
body=None, post_params=None, _preload_content=True,
_request_timeout=None):
"""Perform requests.
:param method: http request method
:param url: http request url
:param query_params: query parameters in the url
:param headers: http request headers
:param body: request json body, for `application/json`
:param post_params: request post parameters,
`application/x-www-form-urlencoded`
and `multipart/form-data`
:param _preload_content: if False, the urllib3.HTTPResponse object will
be returned without reading/decoding response
data. Default is True.
:param _request_timeout: timeout setting for this request. If one
number provided, it will be total request
timeout. It can also be a pair (tuple) of
(connection, read) timeouts.
"""
method = method.upper()
assert method in ['GET', 'HEAD', 'DELETE', 'POST', 'PUT',
'PATCH', 'OPTIONS']
if post_params and body:
raise ApiValueError(
"body parameter cannot be used with post_params parameter."
)
post_params = post_params or {}
headers = headers or {}
timeout = None
if _request_timeout:
if isinstance(_request_timeout, (int, ) if six.PY3 else (int, long)): # noqa: E501,F821
timeout = urllib3.Timeout(total=_request_timeout)
elif (isinstance(_request_timeout, tuple) and
len(_request_timeout) == 2):
timeout = urllib3.Timeout(
connect=_request_timeout[0], read=_request_timeout[1])
if 'Content-Type' not in headers:
headers['Content-Type'] = 'application/json'
try:
# For `POST`, `PUT`, `PATCH`, `OPTIONS`, `DELETE`
if method in ['POST', 'PUT', 'PATCH', 'OPTIONS', 'DELETE']:
if query_params:
url += '?' + urlencode(query_params)
if (re.search('json', headers['Content-Type'], re.IGNORECASE) or
headers['Content-Type'] == 'application/apply-patch+yaml'):
if headers['Content-Type'] == 'application/json-patch+json':
if not isinstance(body, list):
headers['Content-Type'] = \
'application/strategic-merge-patch+json'
request_body = None
if body is not None:
request_body = json.dumps(body)
r = self.pool_manager.request(
method, url,
body=request_body,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
elif headers['Content-Type'] == 'application/x-www-form-urlencoded': # noqa: E501
r = self.pool_manager.request(
method, url,
fields=post_params,
encode_multipart=False,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
elif headers['Content-Type'] == 'multipart/form-data':
# must del headers['Content-Type'], or the correct
# Content-Type which generated by urllib3 will be
# overwritten.
del headers['Content-Type']
r = self.pool_manager.request(
method, url,
fields=post_params,
encode_multipart=True,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
# Pass a `string` parameter directly in the body to support
# other content types than Json when `body` argument is
# provided in serialized form
elif isinstance(body, str) or isinstance(body, bytes):
request_body = body
r = self.pool_manager.request(
method, url,
body=request_body,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
else:
# Cannot generate the request from given parameters
msg = """Cannot prepare a request message for provided
arguments. Please check that your arguments match
declared content type."""
raise ApiException(status=0, reason=msg)
# For `GET`, `HEAD`
else:
r = self.pool_manager.request(method, url,
fields=query_params,
preload_content=_preload_content,
timeout=timeout,
headers=headers)
except urllib3.exceptions.SSLError as e:
msg = "{0}\n{1}".format(type(e).__name__, str(e))
raise ApiException(status=0, reason=msg)
if _preload_content:
r = RESTResponse(r)
# In the python 3, the response.data is bytes.
# we need to decode it to string.
if six.PY3:
r.data = r.data.decode('utf8')
# log response body
logger.debug("response body: %s", r.data)
if not 200 <= r.status <= 299:
> raise ApiException(http_resp=r)
E kubernetes.client.exceptions.ApiException: (500)
E Reason: Internal Server Error
E HTTP response headers: HTTPHeaderDict({'Audit-Id': '37f87163-14c9-4b58-9740-5d495916ee04', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Strict-Transport-Security': 'max-age=31536000; includeSubDomains; preload', 'X-Kubernetes-Pf-Flowschema-Uid': '224291cc-bc8a-47e1-b358-99f64d47f314', 'X-Kubernetes-Pf-Prioritylevel-Uid': '53379352-efa0-40c7-b71a-ad76894c832e', 'Date': 'Thu, 16 Apr 2026 20:29:33 GMT', 'Content-Length': '769'})
E HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: failed calling webhook \"inferenceservice.kserve-webhook-server.defaulter\": failed to call webhook: Post \"https://kserve-webhook-server-service.kserve.svc:443/mutate-serving-kserve-io-v1beta1-inferenceservice?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","reason":"InternalError","details":{"causes":[{"message":"failed calling webhook \"inferenceservice.kserve-webhook-server.defaulter\": failed to call webhook: Post \"https://kserve-webhook-server-service.kserve.svc:443/mutate-serving-kserve-io-v1beta1-inferenceservice?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\""}]},"code":500}
../../python/kserve/.venv/lib64/python3.11/site-packages/kubernetes/client/rest.py:238: ApiException
During handling of the above exception, another exception occurred:
rest_v1_client =
@pytest.mark.graph
@pytest.mark.kourier
@pytest.mark.asyncio(scope="session")
async def test_ig_scenario3(rest_v1_client):
"""
Scenario: Sequence graph with 2 steps - first is hard (and returns non-200) and second is soft dependency.
error_isvc(hard) -> success_isvc (soft)
Expectation: IG will return response of error_isvc and predict_ig will raise exception
"""
logger.info("Starting test test_ig_scenario3")
suffix = str(uuid.uuid4())[1:6]
success_isvc_name, error_isvc_name, success_isvc, error_isvc = setup_isvcs_for_test(
suffix
)
logger.info(f"success_isvc_name is {success_isvc_name}")
logger.info(f"error_isvc_name is {error_isvc_name}")
kserve_client = KServeClient(
config_file=os.environ.get("KUBECONFIG", "~/.kube/config")
)
> kserve_client.create(success_isvc)
graph/test_inference_graph.py:393:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self =
inferenceservice = {'api_version': 'serving.kserve.io/v1beta1',
'kind': 'InferenceService',
'metadata': {'annotations': None,
... 'worker_spec': None,
'xgboost': None},
'transformer': None},
'status': None}
namespace = 'kserve-ci-e2e-test', watch = False, timeout_seconds = 600
def create(
self, inferenceservice, namespace=None, watch=False, timeout_seconds=600
): # pylint:disable=inconsistent-return-statements
"""
Create the inference service
:param inferenceservice: inference service object
:param namespace: defaults to current or default namespace
:param watch: True to watch the created service until timeout elapsed or status is ready
:param timeout_seconds: timeout seconds for watch, default to 600s
:return: created inference service
"""
version = inferenceservice.api_version.split("/")[1]
if namespace is None:
namespace = utils.get_isvc_namespace(inferenceservice)
try:
outputs = self.api_instance.create_namespaced_custom_object(
constants.KSERVE_GROUP,
version,
namespace,
constants.KSERVE_PLURAL_INFERENCESERVICE,
inferenceservice,
)
except client.rest.ApiException as e:
> raise RuntimeError(
"Exception when calling CustomObjectsApi->create_namespaced_custom_object:\
%s\n"
% e
)
E RuntimeError: Exception when calling CustomObjectsApi->create_namespaced_custom_object: (500)
E Reason: Internal Server Error
E HTTP response headers: HTTPHeaderDict({'Audit-Id': '37f87163-14c9-4b58-9740-5d495916ee04', 'Cache-Control': 'no-cache, private', 'Content-Type': 'application/json', 'Strict-Transport-Security': 'max-age=31536000; includeSubDomains; preload', 'X-Kubernetes-Pf-Flowschema-Uid': '224291cc-bc8a-47e1-b358-99f64d47f314', 'X-Kubernetes-Pf-Prioritylevel-Uid': '53379352-efa0-40c7-b71a-ad76894c832e', 'Date': 'Thu, 16 Apr 2026 20:29:33 GMT', 'Content-Length': '769'})
E HTTP response body: {"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: failed calling webhook \"inferenceservice.kserve-webhook-server.defaulter\": failed to call webhook: Post \"https://kserve-webhook-server-service.kserve.svc:443/mutate-serving-kserve-io-v1beta1-inferenceservice?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\"","reason":"InternalError","details":{"causes":[{"message":"failed calling webhook \"inferenceservice.kserve-webhook-server.defaulter\": failed to call webhook: Post \"https://kserve-webhook-server-service.kserve.svc:443/mutate-serving-kserve-io-v1beta1-inferenceservice?timeout=10s\": no endpoints available for service \"kserve-webhook-server-service\""}]},"code":500}
../../python/kserve/kserve/api/kserve_client.py:153: RuntimeError
------------------------------ Captured log call -------------------------------
INFO kserve.trace:test_inference_graph.py:382 Starting test test_ig_scenario3
INFO kserve.trace:test_inference_graph.py:207 SUCCESS_ISVC_IMAGE is quay.io/opendatahub/success-200-isvc:odh-pr-1326
INFO kserve.trace:test_inference_graph.py:208 ERROR_ISVC_IMAGE is quay.io/opendatahub/error-404-isvc:odh-pr-1326
INFO kserve.trace:test_inference_graph.py:387 success_isvc_name is success-200-isvc-aeddd
INFO kserve.trace:test_inference_graph.py:388 error_isvc_name is error-404-isvc-aeddd
=============================== warnings summary ===============================
llmisvc/test_llm_inference_service.py:151
llmisvc/test_llm_inference_service.py:151
/workspace/source/test/e2e/llmisvc/test_llm_inference_service.py:151: PytestUnknownMarkWarning: Unknown pytest.mark.custom_gateway - is this a typo? You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html
pytest.mark.custom_gateway,
llmisvc/test_llm_inference_service.py:200
llmisvc/test_llm_inference_service.py:200
/workspace/source/test/e2e/llmisvc/test_llm_inference_service.py:200: PytestUnknownMarkWarning: Unknown pytest.mark.custom_gateway - is this a typo? You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html
pytest.mark.custom_gateway,
llmisvc/test_llm_inference_service.py:252
llmisvc/test_llm_inference_service.py:252
/workspace/source/test/e2e/llmisvc/test_llm_inference_service.py:252: PytestUnknownMarkWarning: Unknown pytest.mark.custom_gateway - is this a typo? You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html
pytest.mark.custom_gateway,
llmisvc/test_llm_inference_service.py:299
llmisvc/test_llm_inference_service.py:299
/workspace/source/test/e2e/llmisvc/test_llm_inference_service.py:299: PytestUnknownMarkWarning: Unknown pytest.mark.cluster_gpu - is this a typo? You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html
pytest.mark.cluster_gpu,
llmisvc/test_llm_inference_service.py:316
llmisvc/test_llm_inference_service.py:316
/workspace/source/test/e2e/llmisvc/test_llm_inference_service.py:316: PytestUnknownMarkWarning: Unknown pytest.mark.no_scheduler - is this a typo? You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html
pytest.mark.no_scheduler,
llmisvc/test_llm_inference_service.py:329
llmisvc/test_llm_inference_service.py:329
/workspace/source/test/e2e/llmisvc/test_llm_inference_service.py:329: PytestUnknownMarkWarning: Unknown pytest.mark.cluster_multi_node - is this a typo? You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html
marks=[pytest.mark.cluster_cpu, pytest.mark.cluster_multi_node],
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
=========================== short test summary info ============================
FAILED graph/test_inference_graph.py::test_ig_scenario5 - httpx.HTTPStatusErr...
FAILED graph/test_inference_graph.py::test_inference_graph - httpx.HTTPStatus...
FAILED graph/test_inference_graph.py::test_ig_scenario6 - json.decoder.JSONDe...
FAILED graph/test_inference_graph.py::test_ig_scenario1 - json.decoder.JSONDe...
FAILED graph/test_inference_graph.py::test_ig_scenario7 - httpx.ConnectError:...
FAILED graph/test_inference_graph.py::test_ig_scenario8 - RuntimeError: Excep...
FAILED graph/test_inference_graph.py::test_ig_scenario9 - RuntimeError: Excep...
FAILED graph/test_inference_graph.py::test_ig_scenario10 - RuntimeError: Exce...
FAILED graph/test_inference_graph.py::test_ig_scenario4 - RuntimeError: Excep...
FAILED graph/test_inference_graph.py::test_ig_scenario2 - httpx.ConnectError:...
FAILED graph/test_inference_graph.py::test_ig_scenario3 - RuntimeError: Excep...
=========== 11 failed, 3 skipped, 12 warnings in 6653.02s (1:50:53) ============
[must-gather ] OUT 2026-04-16T20:29:34.608441037Z Using must-gather plug-in image: quay.io/modh/must-gather:rhoai-2.24
When opening a support case, bugzilla, or issue please include the following summary data along with any other requested information:
ClusterID: 67c06221-5186-43de-b2c1-af0e11a133d6
ClientVersion: 4.21.5
ClusterVersion: Stable at "4.20.18"
ClusterOperators:
clusteroperator/dns is not available (DNS "default" is unavailable.) because DNS default is degraded
clusteroperator/image-registry is not available (Available: The registry is removed
NodeCADaemonAvailable: The daemon set node-ca does not have available replicas
ImagePrunerAvailable: Pruner CronJob has been created) because Degraded: The registry is removed
clusteroperator/ingress is not available (The "default" ingress controller does not exist.) because The "default" ingress controller does not exist.
clusteroperator/network is progressing: Deployment "/openshift-network-console/networking-console-plugin" is not available (awaiting 1 nodes)
clusteroperator/node-tuning is not available (DaemonSet "tuned" has no available Pod(s)) because DaemonSet "tuned" available
clusteroperator/storage is not available (AWSEBSCSIDriverOperatorCRAvailable: AWSEBSDriverNodeServiceControllerAvailable: Waiting for the DaemonSet to deploy the CSI Node Service) because AWSEBSCSIDriverOperatorCRDegraded: All is well
clusteroperator/authentication is missing
clusteroperator/cloud-credential is missing
clusteroperator/cluster-autoscaler is missing
clusteroperator/config-operator is missing
clusteroperator/etcd is missing
clusteroperator/machine-api is missing
clusteroperator/machine-approver is missing
clusteroperator/machine-config is missing
clusteroperator/marketplace is missing
[must-gather ] OUT 2026-04-16T20:29:34.659690089Z namespace/openshift-must-gather-tqjz6 created
[must-gather ] OUT 2026-04-16T20:29:34.666719814Z clusterrolebinding.rbac.authorization.k8s.io/must-gather-rnjmn created
[must-gather ] OUT 2026-04-16T20:29:34.688037053Z namespace/openshift-must-gather-tqjz6 deleted
Reprinting Cluster State:
When opening a support case, bugzilla, or issue please include the following summary data along with any other requested information:
ClusterID: 67c06221-5186-43de-b2c1-af0e11a133d6
ClientVersion: 4.21.5
ClusterVersion: Stable at "4.20.18"
ClusterOperators:
clusteroperator/dns is not available (DNS "default" is unavailable.) because DNS default is degraded
clusteroperator/image-registry is not available (Available: The registry is removed
NodeCADaemonAvailable: The daemon set node-ca does not have available replicas
ImagePrunerAvailable: Pruner CronJob has been created) because Degraded: The registry is removed
clusteroperator/ingress is not available (The "default" ingress controller does not exist.) because The "default" ingress controller does not exist.
clusteroperator/network is progressing: Deployment "/openshift-network-console/networking-console-plugin" is not available (awaiting 1 nodes)
clusteroperator/node-tuning is not available (DaemonSet "tuned" has no available Pod(s)) because DaemonSet "tuned" available
clusteroperator/storage is not available (AWSEBSCSIDriverOperatorCRAvailable: AWSEBSDriverNodeServiceControllerAvailable: Waiting for the DaemonSet to deploy the CSI Node Service) because AWSEBSCSIDriverOperatorCRDegraded: All is well
clusteroperator/authentication is missing
clusteroperator/cloud-credential is missing
clusteroperator/cluster-autoscaler is missing
clusteroperator/config-operator is missing
clusteroperator/etcd is missing
clusteroperator/machine-api is missing
clusteroperator/machine-approver is missing
clusteroperator/machine-config is missing
clusteroperator/marketplace is missing
Error from server (InternalError): Internal error occurred: failed calling webhook "block-resources.hypershift.openshift.io": failed to call webhook: Post "https://xxx-invalid-service-xxx.default.svc:443/validate?timeout=30s": service "xxx-invalid-service-xxx" not found
Failing pipeline because deploy-and-e2e step failed
clean-cut/oauth-config-struct
https://github.com/bartoszmajsak/kserve
Cloning into '/workspace/source'...
WORK_DIR: /workspace/odh-ci-artifacts
REPO_PATH: opendatahub-io/odh-build-metadata
REPO_BRANCH: ci-artifacts
SPARSE_FILE_PATH: test-artifacts/docs
SOURCE_PATH: /workspace/artifacts-dir
DEST_PATH: test-artifacts/kserve-group-test-8mlhc
ALWAYS_PASS: false
configuring gh token
taking github token from Konflux bot
Initialized empty Git repository in /workspace/odh-ci-artifacts/.git/
From https://github.com/opendatahub-io/odh-build-metadata
* branch ci-artifacts -> FETCH_HEAD
* [new branch] ci-artifacts -> origin/ci-artifacts
Already on 'ci-artifacts'
branch 'ci-artifacts' set up to track 'origin/ci-artifacts'.
TASK_NAME=kserve-group-test-8mlhc-e2e-graph
PIPELINERUN_NAME=kserve-group-test-8mlhc
From https://github.com/opendatahub-io/odh-build-metadata
* branch ci-artifacts -> FETCH_HEAD
7f89eaa..16ebdf1 ci-artifacts -> origin/ci-artifacts
Updating 7f89eaa..16ebdf1
Fast-forward
.../e2e-llm-inference-service.tar.gz | Bin 0 -> 14826029 bytes
1 file changed, 0 insertions(+), 0 deletions(-)
create mode 100644 test-artifacts/kserve-group-test-ldcnp/e2e-llm-inference-service.tar.gz
-rw-r--r--. 1 root 1001540000 2665 Apr 16 20:31 /workspace/odh-ci-artifacts/test-artifacts/kserve-group-test-8mlhc/e2e-graph.tar.gz
[ci-artifacts 897db6c] Updating CI Artifacts in e2e-graph
1 file changed, 0 insertions(+), 0 deletions(-)
create mode 100644 test-artifacts/kserve-group-test-8mlhc/e2e-graph.tar.gz
From https://github.com/opendatahub-io/odh-build-metadata
* branch ci-artifacts -> FETCH_HEAD
Already up to date.
To https://github.com/opendatahub-io/odh-build-metadata.git
16ebdf1..897db6c ci-artifacts -> ci-artifacts
2026/04/16 18:35:10 Entrypoint initialization
2026/04/16 18:35:10 Decoded script /tekton/scripts/script-0-g6rr8
2026/04/16 18:35:10 Decoded script /tekton/scripts/script-1-qd9kl
2026/04/16 18:35:10 Decoded script /tekton/scripts/script-2-nxlp9
2026/04/16 18:35:10 Decoded script /tekton/scripts/script-3-kdzjw
2026/04/16 18:35:10 Decoded script /tekton/scripts/script-4-f8fs7
2026/04/16 18:35:10 Decoded script /tekton/scripts/script-5-xsj9w
Found kubeconfig secret: cluster-hxxv8-admin-kubeconfig
Wrote kubeconfig to /credentials/cluster-hxxv8-kubeconfig
Found admin password secret: cluster-hxxv8-admin-password
Retrieved username
Wrote password to /credentials/cluster-hxxv8-password
API Server URL: https://a9b56e18e9e4c45dcaeda4bb15f69bb8-9f16fbcdf636ba83.elb.us-east-1.amazonaws.com:6443
Console URL: https://console-openshift-console.apps.bfe2dca5-bbfe-47e9-bd3f-ac922f58bfea.prod.konfluxeaas.com