Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 361, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 250, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 170, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 536, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1710, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 806, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 630, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 726, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1232, in execute result: FixtureValue = ihook.pytest_fixture_setup( ~~~~~~~~~~~~~~~~~~~~~~~~~~^ fixturedef=self, request=request ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pytest_asyncio/plugin.py", line 924, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper return result.get_result() ~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result raise exc.with_traceback(tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper res = yield ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/setuponly.py", line 36, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1328, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 998, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_runtime/openvino/conftest.py", line 140, in openvino_inference_service with create_isvc(**service_config) as isvc: ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'onnx-predictor-56f8756f48-qjzgn': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:10:52Z', 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:10:50Z', 'message': 'containers with incomplete status: [storage-initializer]', 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:10:50Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:10:50Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:10:50Z', 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'registry.redhat.io/rhoai/odh-openvino-model-server-rhel9@sha256:5bb57ef927e7481cfc768993faa8e1e70b27154ecc81a88991bd5428001c1ea6', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-w79kr', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}, {'image': 'registry.redhat.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:84459a75f19a225c932cdff52fd81f21e8cacc1fc48b236b0b82e2f1412da37e', 'imageID': '', 'lastState': {}, 'name': 'kube-rbac-proxy', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/etc/kube-rbac-proxy', 'name': 'onnx-kube-rbac-proxy-sar-config', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-w79kr', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '192.169.1.253', 'hostIPs': [{'ip': '192.169.1.253'}], 'initContainerStatuses': [{'containerID': 'cri-o://f2c9760d8e1d5eacf80cf916be6c1fbb4ea8122aa296864aa124cce0ef794f3e', 'image': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:0cf99daf3963a5b1ef859b6b54b0516e8cb2117874390989889db169ac387a86', 'imageID': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:0cf99daf3963a5b1ef859b6b54b0516e8cb2117874390989889db169ac387a86', 'lastState': {'terminated': {'containerID': 'cri-o://f2c9760d8e1d5eacf80cf916be6c1fbb4ea8122aa296864aa124cce0ef794f3e', 'exitCode': 1, 'finishedAt': '2026-07-09T16:10:54Z', 'message': '2026-07-09 16:10:53.797 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' '(src_uri, dest_path): ' "[('s3://mlflow-e2e/openvino/model_repository/onnx/', " "'/mnt/models')]\n" '2026-07-09 16:10:53.797 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://mlflow-e2e/openvino/model_repository/onnx/ to local\n' '/prod_venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1097: ' 'InsecureRequestWarning: Unverified HTTPS request is being made to ' "host 's3.amazonaws.com'. Adding certificate verification is " 'strongly advised. See: ' 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings\n' ' warnings.warn(\n' '/prod_venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1097: ' 'InsecureRequestWarning: Unverified HTTPS request is being made to ' "host 's3.amazonaws.com'. Adding certificate verification is " 'strongly advised. See: ' 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings\n' ' warnings.warn(\n' '2026-07-09 16:10:54.393 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():21] Storage initialization ' 'failed: Failed to fetch model. No model found in ' 'openvino/model_repository/onnx/.\n', 'reason': 'Error', 'startedAt': '2026-07-09T16:10:53Z'}}, 'name': 'storage-initializer', 'ready': False, 'restartCount': 1, 'started': False, 'state': {'waiting': {'message': 'back-off 10s restarting failed container=storage-initializer ' 'pod=onnx-predictor-56f8756f48-qjzgn_onnx-standard-rest(9ff6cfd1-45d8-49e7-a2e8-f2dba08d70e7)', 'reason': 'CrashLoopBackOff'}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-w79kr', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'phase': 'Pending', 'podIP': '10.128.2.154', 'podIPs': [{'ip': '10.128.2.154'}], 'qosClass': 'Burstable', 'startTime': '2026-07-09T16:10:50Z'}}Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 361, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 250, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 170, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 536, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1710, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 806, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 630, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 726, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1181, in execute fixturedef = request._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 726, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1232, in execute result: FixtureValue = ihook.pytest_fixture_setup( ~~~~~~~~~~~~~~~~~~~~~~~~~~^ fixturedef=self, request=request ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pytest_asyncio/plugin.py", line 924, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper return result.get_result() ~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result raise exc.with_traceback(tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper res = yield ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/setuponly.py", line 36, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1328, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 998, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_runtime/triton/S3/conftest.py", line 157, in triton_serving_runtime with ServingRuntimeFromTemplate( ~~~~~~~~~~~~~~~~~~~~~~~~~~^ client=admin_client, ^^^^^^^^^^^^^^^^^^^^ ...<3 lines>... deployment_type=request.param.get("deployment_mode", KServeDeploymentType.STANDARD), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) as model_runtime: ^ File "/home/odh/opendatahub-tests/utilities/serving_runtime.py", line 132, in __init__ self.model_dict = self.update_model_dict() ~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/utilities/serving_runtime.py", line 185, in update_model_dict _model_dict = self.get_model_dict_from_template() File "/home/odh/opendatahub-tests/utilities/serving_runtime.py", line 170, in get_model_dict_from_template template = self.get_model_template() File "/home/odh/opendatahub-tests/utilities/serving_runtime.py", line 160, in get_model_template raise ResourceNotFoundError(f"{self.template_name} template not found") kubernetes.dynamic.exceptions.ResourceNotFoundError: triton-rest-runtime-template template not foundTraceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 361, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 250, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 170, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 536, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1710, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 806, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 630, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 726, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1232, in execute result: FixtureValue = ihook.pytest_fixture_setup( ~~~~~~~~~~~~~~~~~~~~~~~~~~^ fixturedef=self, request=request ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pytest_asyncio/plugin.py", line 924, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper return result.get_result() ~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result raise exc.with_traceback(tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper res = yield ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/setuponly.py", line 36, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1328, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 998, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_runtime/triton/S3/conftest.py", line 212, in triton_inference_service with create_isvc(**service_config) as isvc: ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'tensorflow-standard-rest-predictor-fc576768b-t9tb9': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:12:01Z', 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:12:00Z', 'message': 'containers with incomplete status: [storage-initializer]', 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:12:00Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:12:00Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:12:00Z', 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'nvcr.io/nvidia/tritonserver:25.02-py3', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-76g5t', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}, {'image': 'registry.redhat.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:84459a75f19a225c932cdff52fd81f21e8cacc1fc48b236b0b82e2f1412da37e', 'imageID': '', 'lastState': {}, 'name': 'kube-rbac-proxy', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/etc/kube-rbac-proxy', 'name': 'tensorflow-standard-rest-kube-rbac-proxy-sar-config', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-76g5t', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '192.169.3.92', 'hostIPs': [{'ip': '192.169.3.92'}], 'initContainerStatuses': [{'containerID': 'cri-o://b5cfec9fb559daa32762cdc248c6a4e4fcf9b0043f685705c1b78b756f58ec38', 'image': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:0cf99daf3963a5b1ef859b6b54b0516e8cb2117874390989889db169ac387a86', 'imageID': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:0cf99daf3963a5b1ef859b6b54b0516e8cb2117874390989889db169ac387a86', 'lastState': {'terminated': {'containerID': 'cri-o://b5cfec9fb559daa32762cdc248c6a4e4fcf9b0043f685705c1b78b756f58ec38', 'exitCode': 1, 'finishedAt': '2026-07-09T16:12:03Z', 'message': '2026-07-09 16:12:02.572 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' '(src_uri, dest_path): ' "[('s3://mlflow-e2e/triton/model_repository/', '/mnt/models')]\n" '2026-07-09 16:12:02.572 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://mlflow-e2e/triton/model_repository/ to local\n' '/prod_venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1097: ' 'InsecureRequestWarning: Unverified HTTPS request is being made to ' "host 's3.amazonaws.com'. Adding certificate verification is " 'strongly advised. See: ' 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings\n' ' warnings.warn(\n' '/prod_venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1097: ' 'InsecureRequestWarning: Unverified HTTPS request is being made to ' "host 's3.amazonaws.com'. Adding certificate verification is " 'strongly advised. See: ' 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings\n' ' warnings.warn(\n' '2026-07-09 16:12:03.112 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():21] Storage initialization ' 'failed: Failed to fetch model. No model found in ' 'triton/model_repository/.\n', 'reason': 'Error', 'startedAt': '2026-07-09T16:12:02Z'}}, 'name': 'storage-initializer', 'ready': False, 'restartCount': 1, 'started': False, 'state': {'waiting': {'message': 'back-off 10s restarting failed container=storage-initializer ' 'pod=tensorflow-standard-rest-predictor-fc576768b-t9tb9_tensorflow-standard(042b473e-1597-4f97-96f5-ba5f1b6ac6a1)', 'reason': 'CrashLoopBackOff'}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-76g5t', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'phase': 'Pending', 'podIP': '10.131.0.79', 'podIPs': [{'ip': '10.131.0.79'}], 'qosClass': 'Burstable', 'startTime': '2026-07-09T16:12:00Z'}}Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 361, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 250, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 170, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 536, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1710, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 806, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 630, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 726, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1232, in execute result: FixtureValue = ihook.pytest_fixture_setup( ~~~~~~~~~~~~~~~~~~~~~~~~~~^ fixturedef=self, request=request ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pytest_asyncio/plugin.py", line 924, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper return result.get_result() ~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result raise exc.with_traceback(tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper res = yield ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/setuponly.py", line 36, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1328, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 998, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_runtime/vllm/cpu/cpu_x86/conftest.py", line 101, in cpu_x86_inference_service with create_isvc(**isvc_kwargs) as isvc: ~~~~~~~~~~~^^^^^^^^^^^^^^^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'opt-125m-standard-cpu-predictor-5bf786d769-xjdrc': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:12:22Z', 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:12:21Z', 'message': 'containers with incomplete status: [storage-initializer]', 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:12:21Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:12:21Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:12:21Z', 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'registry.redhat.io/rhaii/vllm-cpu-rhel9@sha256:562aeaf7dabfccccf9ecc2d8c0e7d84d813778f146dd9d5c8e3a1d595d07a82c', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/dev/shm', 'name': 'shared-memory'}, {'mountPath': '/tmp', 'name': 'tmp'}, {'mountPath': '/home/vllm', 'name': 'home'}, {'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-f6xg4', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}, {'image': 'registry.redhat.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:84459a75f19a225c932cdff52fd81f21e8cacc1fc48b236b0b82e2f1412da37e', 'imageID': '', 'lastState': {}, 'name': 'kube-rbac-proxy', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/etc/kube-rbac-proxy', 'name': 'opt-125m-standard-cpu-kube-rbac-proxy-sar-config', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-f6xg4', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '192.169.1.253', 'hostIPs': [{'ip': '192.169.1.253'}], 'initContainerStatuses': [{'containerID': 'cri-o://4e26dd818b084ee736b4d4adb37b837d6f36082a2ad37dff0caf7cdbec4ca4c1', 'image': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:0cf99daf3963a5b1ef859b6b54b0516e8cb2117874390989889db169ac387a86', 'imageID': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:0cf99daf3963a5b1ef859b6b54b0516e8cb2117874390989889db169ac387a86', 'lastState': {'terminated': {'containerID': 'cri-o://4e26dd818b084ee736b4d4adb37b837d6f36082a2ad37dff0caf7cdbec4ca4c1', 'exitCode': 1, 'finishedAt': '2026-07-09T16:12:24Z', 'message': '2026-07-09 16:12:24.160 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' "(src_uri, dest_path): [('s3://mlflow-e2e/opt-125m/', " "'/mnt/models')]\n" '2026-07-09 16:12:24.160 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://mlflow-e2e/opt-125m/ to local\n' '/prod_venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1097: ' 'InsecureRequestWarning: Unverified HTTPS request is being made to ' "host 's3.amazonaws.com'. Adding certificate verification is " 'strongly advised. See: ' 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings\n' ' warnings.warn(\n' '/prod_venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1097: ' 'InsecureRequestWarning: Unverified HTTPS request is being made to ' "host 's3.amazonaws.com'. Adding certificate verification is " 'strongly advised. See: ' 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings\n' ' warnings.warn(\n' '2026-07-09 16:12:24.799 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():21] Storage initialization ' 'failed: Failed to fetch model. No model found in opt-125m/.\n', 'reason': 'Error', 'startedAt': '2026-07-09T16:12:23Z'}}, 'name': 'storage-initializer', 'ready': False, 'restartCount': 1, 'started': False, 'state': {'waiting': {'message': 'back-off 10s restarting failed container=storage-initializer ' 'pod=opt-125m-standard-cpu-predictor-5bf786d769-xjdrc_opt-125m-standard-cpu(d5129dee-8620-4381-bfb0-3312ee298143)', 'reason': 'CrashLoopBackOff'}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-f6xg4', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'phase': 'Pending', 'podIP': '10.128.2.155', 'podIPs': [{'ip': '10.128.2.155'}], 'qosClass': 'Burstable', 'startTime': '2026-07-09T16:12:21Z'}}Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 361, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 250, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 170, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 536, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1710, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 806, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 630, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 726, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1232, in execute result: FixtureValue = ihook.pytest_fixture_setup( ~~~~~~~~~~~~~~~~~~~~~~~~~~^ fixturedef=self, request=request ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pytest_asyncio/plugin.py", line 924, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper return result.get_result() ~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result raise exc.with_traceback(tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper res = yield ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/setuponly.py", line 36, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1328, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 998, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_runtime/vllm/probes/conftest.py", line 108, in vllm_probes_inference_service with create_isvc(**isvc_kwargs) as isvc: ~~~~~~~~~~~^^^^^^^^^^^^^^^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'opt-125m-probes-predictor-85b56667d6-5g56k': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:12:43Z', 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:12:42Z', 'message': 'containers with incomplete status: [storage-initializer]', 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:12:42Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:12:42Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:12:42Z', 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'registry.redhat.io/rhaii/vllm-cpu-rhel9@sha256:562aeaf7dabfccccf9ecc2d8c0e7d84d813778f146dd9d5c8e3a1d595d07a82c', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/dev/shm', 'name': 'shared-memory'}, {'mountPath': '/tmp', 'name': 'tmp'}, {'mountPath': '/home/vllm', 'name': 'home'}, {'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-dngwp', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}, {'image': 'registry.redhat.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:84459a75f19a225c932cdff52fd81f21e8cacc1fc48b236b0b82e2f1412da37e', 'imageID': '', 'lastState': {}, 'name': 'kube-rbac-proxy', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/etc/kube-rbac-proxy', 'name': 'opt-125m-probes-kube-rbac-proxy-sar-config', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-dngwp', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '192.169.1.253', 'hostIPs': [{'ip': '192.169.1.253'}], 'initContainerStatuses': [{'containerID': 'cri-o://ba3ca26d7f1d683a3193c9b140bbe11fb28174d97efee1daa0183a6253965c4d', 'image': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:0cf99daf3963a5b1ef859b6b54b0516e8cb2117874390989889db169ac387a86', 'imageID': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:0cf99daf3963a5b1ef859b6b54b0516e8cb2117874390989889db169ac387a86', 'lastState': {'terminated': {'containerID': 'cri-o://ba3ca26d7f1d683a3193c9b140bbe11fb28174d97efee1daa0183a6253965c4d', 'exitCode': 1, 'finishedAt': '2026-07-09T16:12:45Z', 'message': '2026-07-09 16:12:45.211 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' "(src_uri, dest_path): [('s3://mlflow-e2e/opt-125m/', " "'/mnt/models')]\n" '2026-07-09 16:12:45.212 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://mlflow-e2e/opt-125m/ to local\n' '/prod_venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1097: ' 'InsecureRequestWarning: Unverified HTTPS request is being made to ' "host 's3.amazonaws.com'. Adding certificate verification is " 'strongly advised. See: ' 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings\n' ' warnings.warn(\n' '/prod_venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1097: ' 'InsecureRequestWarning: Unverified HTTPS request is being made to ' "host 's3.amazonaws.com'. Adding certificate verification is " 'strongly advised. See: ' 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings\n' ' warnings.warn(\n' '2026-07-09 16:12:45.819 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():21] Storage initialization ' 'failed: Failed to fetch model. No model found in opt-125m/.\n', 'reason': 'Error', 'startedAt': '2026-07-09T16:12:44Z'}}, 'name': 'storage-initializer', 'ready': False, 'restartCount': 1, 'started': False, 'state': {'waiting': {'message': 'back-off 10s restarting failed container=storage-initializer ' 'pod=opt-125m-probes-predictor-85b56667d6-5g56k_opt-125m-probes(eb669508-7eaa-4d69-b3fc-c8f9a41961a8)', 'reason': 'CrashLoopBackOff'}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-dngwp', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'phase': 'Pending', 'podIP': '10.128.2.156', 'podIPs': [{'ip': '10.128.2.156'}], 'qosClass': 'Burstable', 'startTime': '2026-07-09T16:12:42Z'}}Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 361, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 250, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 170, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 536, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1710, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 806, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 630, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 726, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1204, in execute raise exc.with_traceback(exc_tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1328, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 998, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_runtime/vllm/probes/conftest.py", line 108, in vllm_probes_inference_service with create_isvc(**isvc_kwargs) as isvc: ~~~~~~~~~~~^^^^^^^^^^^^^^^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'opt-125m-probes-predictor-85b56667d6-5g56k': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:12:43Z', 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:12:42Z', 'message': 'containers with incomplete status: [storage-initializer]', 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:12:42Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:12:42Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T16:12:42Z', 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'registry.redhat.io/rhaii/vllm-cpu-rhel9@sha256:562aeaf7dabfccccf9ecc2d8c0e7d84d813778f146dd9d5c8e3a1d595d07a82c', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/dev/shm', 'name': 'shared-memory'}, {'mountPath': '/tmp', 'name': 'tmp'}, {'mountPath': '/home/vllm', 'name': 'home'}, {'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-dngwp', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}, {'image': 'registry.redhat.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:84459a75f19a225c932cdff52fd81f21e8cacc1fc48b236b0b82e2f1412da37e', 'imageID': '', 'lastState': {}, 'name': 'kube-rbac-proxy', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/etc/kube-rbac-proxy', 'name': 'opt-125m-probes-kube-rbac-proxy-sar-config', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-dngwp', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '192.169.1.253', 'hostIPs': [{'ip': '192.169.1.253'}], 'initContainerStatuses': [{'containerID': 'cri-o://ba3ca26d7f1d683a3193c9b140bbe11fb28174d97efee1daa0183a6253965c4d', 'image': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:0cf99daf3963a5b1ef859b6b54b0516e8cb2117874390989889db169ac387a86', 'imageID': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:0cf99daf3963a5b1ef859b6b54b0516e8cb2117874390989889db169ac387a86', 'lastState': {'terminated': {'containerID': 'cri-o://ba3ca26d7f1d683a3193c9b140bbe11fb28174d97efee1daa0183a6253965c4d', 'exitCode': 1, 'finishedAt': '2026-07-09T16:12:45Z', 'message': '2026-07-09 16:12:45.211 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' "(src_uri, dest_path): [('s3://mlflow-e2e/opt-125m/', " "'/mnt/models')]\n" '2026-07-09 16:12:45.212 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://mlflow-e2e/opt-125m/ to local\n' '/prod_venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1097: ' 'InsecureRequestWarning: Unverified HTTPS request is being made to ' "host 's3.amazonaws.com'. Adding certificate verification is " 'strongly advised. See: ' 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings\n' ' warnings.warn(\n' '/prod_venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1097: ' 'InsecureRequestWarning: Unverified HTTPS request is being made to ' "host 's3.amazonaws.com'. Adding certificate verification is " 'strongly advised. See: ' 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings\n' ' warnings.warn(\n' '2026-07-09 16:12:45.819 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():21] Storage initialization ' 'failed: Failed to fetch model. No model found in opt-125m/.\n', 'reason': 'Error', 'startedAt': '2026-07-09T16:12:44Z'}}, 'name': 'storage-initializer', 'ready': False, 'restartCount': 1, 'started': False, 'state': {'waiting': {'message': 'back-off 10s restarting failed container=storage-initializer ' 'pod=opt-125m-probes-predictor-85b56667d6-5g56k_opt-125m-probes(eb669508-7eaa-4d69-b3fc-c8f9a41961a8)', 'reason': 'CrashLoopBackOff'}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-dngwp', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'phase': 'Pending', 'podIP': '10.128.2.156', 'podIPs': [{'ip': '10.128.2.156'}], 'qosClass': 'Burstable', 'startTime': '2026-07-09T16:12:42Z'}}