Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 361, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 250, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 170, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 536, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1710, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 806, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 630, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 726, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1232, in execute result: FixtureValue = ihook.pytest_fixture_setup( ~~~~~~~~~~~~~~~~~~~~~~~~~~^ fixturedef=self, request=request ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pytest_asyncio/plugin.py", line 924, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper return result.get_result() ~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result raise exc.with_traceback(tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper res = yield ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/setuponly.py", line 36, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1328, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 998, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_runtime/openvino/conftest.py", line 140, in openvino_inference_service with create_isvc(**service_config) as isvc: ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'onnx-predictor-7f7b467c8f-m2gnf': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:32:00Z', 'observedGeneration': 1, 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:31:59Z', 'message': 'containers with incomplete status: [storage-initializer]', 'observedGeneration': 1, 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:31:59Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'observedGeneration': 1, 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:31:59Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'observedGeneration': 1, 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:31:59Z', 'observedGeneration': 1, 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'quay.io/rhoai/odh-openvino-model-server-rhel9@sha256:8c1b225509bfdc43756195f34a356d1e2ebd58fcd83fda3d37380dd7f5304df8', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-7dvsb', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}, {'image': 'quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:1e614e46799fd020ac18d95c06a463421154ce86e59ccb3fa6007e66fed9910e', 'imageID': '', 'lastState': {}, 'name': 'kube-rbac-proxy', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/etc/kube-rbac-proxy', 'name': 'kube-rbac-proxy-sar-config', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-7dvsb', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '10.0.1.76', 'hostIPs': [{'ip': '10.0.1.76'}], 'initContainerStatuses': [{'allocatedResources': {'cpu': '100m', 'memory': '100Mi'}, 'containerID': 'cri-o://7796b3500ab7442f00c4925307559974b8499cdad057592396ef45dfc6607485', 'image': 'quay.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:166184c520c7821bacbedd05736938b58edc9cc2edc6071bfbc1105a4316aef5', 'imageID': 'quay.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:166184c520c7821bacbedd05736938b58edc9cc2edc6071bfbc1105a4316aef5', 'lastState': {'terminated': {'containerID': 'cri-o://7796b3500ab7442f00c4925307559974b8499cdad057592396ef45dfc6607485', 'exitCode': 1, 'finishedAt': '2026-07-09T15:32:02Z', 'message': '2026-07-09 15:32:01.440 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():17] Initializing, args: ' '(src_uri, dest_path): ' "[('s3://mlflow-e2e/openvino/model_repository/onnx/', " "'/mnt/models')]\n" '2026-07-09 15:32:01.440 1 storage.initializer INFO ' '[kserve_storage.py:download():166] Copying contents of ' 's3://mlflow-e2e/openvino/model_repository/onnx/ to local\n' '/prod_venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1097: ' 'InsecureRequestWarning: Unverified HTTPS request is being made to ' "host 's3.amazonaws.com'. Adding certificate verification is " 'strongly advised. See: ' 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings\n' ' warnings.warn(\n' '/prod_venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1097: ' 'InsecureRequestWarning: Unverified HTTPS request is being made to ' "host 's3.amazonaws.com'. Adding certificate verification is " 'strongly advised. See: ' 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings\n' ' warnings.warn(\n' '2026-07-09 15:32:02.029 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():28] Storage initialization ' 'failed: Failed to fetch model. No model found in ' 'openvino/model_repository/onnx/.\n', 'reason': 'Error', 'startedAt': '2026-07-09T15:32:01Z'}}, 'name': 'storage-initializer', 'ready': False, 'resources': {'limits': {'cpu': '1', 'memory': '24Gi'}, 'requests': {'cpu': '100m', 'memory': '100Mi'}}, 'restartCount': 1, 'started': False, 'state': {'waiting': {'message': 'back-off 10s restarting failed container=storage-initializer ' 'pod=onnx-predictor-7f7b467c8f-m2gnf_onnx-standard-rest(ed73a190-6047-4884-a13f-e0a8887e3861)', 'reason': 'CrashLoopBackOff'}}, 'user': {'linux': {'gid': 0, 'supplementalGroups': [0, 1001540000], 'uid': 1001540000}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-7dvsb', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'observedGeneration': 1, 'phase': 'Pending', 'podIP': '10.128.0.249', 'podIPs': [{'ip': '10.128.0.249'}], 'qosClass': 'Burstable', 'startTime': '2026-07-09T15:31:59Z'}}Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 361, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 250, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 170, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 536, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1710, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 806, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 630, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 726, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1181, in execute fixturedef = request._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 726, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1232, in execute result: FixtureValue = ihook.pytest_fixture_setup( ~~~~~~~~~~~~~~~~~~~~~~~~~~^ fixturedef=self, request=request ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pytest_asyncio/plugin.py", line 924, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper return result.get_result() ~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result raise exc.with_traceback(tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper res = yield ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/setuponly.py", line 36, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1328, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 998, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_runtime/triton/S3/conftest.py", line 157, in triton_serving_runtime with ServingRuntimeFromTemplate( ~~~~~~~~~~~~~~~~~~~~~~~~~~^ client=admin_client, ^^^^^^^^^^^^^^^^^^^^ ...<3 lines>... deployment_type=request.param.get("deployment_mode", KServeDeploymentType.STANDARD), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) as model_runtime: ^ File "/home/odh/opendatahub-tests/utilities/serving_runtime.py", line 132, in __init__ self.model_dict = self.update_model_dict() ~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/utilities/serving_runtime.py", line 185, in update_model_dict _model_dict = self.get_model_dict_from_template() File "/home/odh/opendatahub-tests/utilities/serving_runtime.py", line 170, in get_model_dict_from_template template = self.get_model_template() File "/home/odh/opendatahub-tests/utilities/serving_runtime.py", line 160, in get_model_template raise ResourceNotFoundError(f"{self.template_name} template not found") kubernetes.dynamic.exceptions.ResourceNotFoundError: triton-rest-runtime-template template not foundTraceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 361, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 250, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 170, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 536, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1710, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 806, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 630, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 726, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1232, in execute result: FixtureValue = ihook.pytest_fixture_setup( ~~~~~~~~~~~~~~~~~~~~~~~~~~^ fixturedef=self, request=request ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pytest_asyncio/plugin.py", line 924, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper return result.get_result() ~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result raise exc.with_traceback(tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper res = yield ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/setuponly.py", line 36, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1328, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 998, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_runtime/triton/S3/conftest.py", line 212, in triton_inference_service with create_isvc(**service_config) as isvc: ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'tensorflow-standard-rest-predictor-5cf5fcc4cc-kntz5': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:33:15Z', 'observedGeneration': 1, 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:33:14Z', 'message': 'containers with incomplete status: [storage-initializer]', 'observedGeneration': 1, 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:33:14Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'observedGeneration': 1, 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:33:14Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'observedGeneration': 1, 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:33:14Z', 'observedGeneration': 1, 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'nvcr.io/nvidia/tritonserver:25.02-py3', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-l74j8', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}, {'image': 'quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:1e614e46799fd020ac18d95c06a463421154ce86e59ccb3fa6007e66fed9910e', 'imageID': '', 'lastState': {}, 'name': 'kube-rbac-proxy', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/etc/kube-rbac-proxy', 'name': 'kube-rbac-proxy-sar-config', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-l74j8', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '10.0.1.76', 'hostIPs': [{'ip': '10.0.1.76'}], 'initContainerStatuses': [{'allocatedResources': {'cpu': '100m', 'memory': '100Mi'}, 'containerID': 'cri-o://3c506312d8f8257af3499a1e8e6e378061605a2628c74cfd066f8a131351f5f8', 'image': 'quay.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:166184c520c7821bacbedd05736938b58edc9cc2edc6071bfbc1105a4316aef5', 'imageID': 'quay.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:166184c520c7821bacbedd05736938b58edc9cc2edc6071bfbc1105a4316aef5', 'lastState': {'terminated': {'containerID': 'cri-o://3c506312d8f8257af3499a1e8e6e378061605a2628c74cfd066f8a131351f5f8', 'exitCode': 1, 'finishedAt': '2026-07-09T15:33:17Z', 'message': '2026-07-09 15:33:16.934 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():17] Initializing, args: ' '(src_uri, dest_path): ' "[('s3://mlflow-e2e/triton/model_repository/', '/mnt/models')]\n" '2026-07-09 15:33:16.935 1 storage.initializer INFO ' '[kserve_storage.py:download():166] Copying contents of ' 's3://mlflow-e2e/triton/model_repository/ to local\n' '/prod_venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1097: ' 'InsecureRequestWarning: Unverified HTTPS request is being made to ' "host 's3.amazonaws.com'. Adding certificate verification is " 'strongly advised. See: ' 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings\n' ' warnings.warn(\n' '/prod_venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1097: ' 'InsecureRequestWarning: Unverified HTTPS request is being made to ' "host 's3.amazonaws.com'. Adding certificate verification is " 'strongly advised. See: ' 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings\n' ' warnings.warn(\n' '2026-07-09 15:33:17.591 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():28] Storage initialization ' 'failed: Failed to fetch model. No model found in ' 'triton/model_repository/.\n', 'reason': 'Error', 'startedAt': '2026-07-09T15:33:16Z'}}, 'name': 'storage-initializer', 'ready': False, 'resources': {'limits': {'cpu': '1', 'memory': '24Gi'}, 'requests': {'cpu': '100m', 'memory': '100Mi'}}, 'restartCount': 1, 'started': False, 'state': {'waiting': {'message': 'back-off 10s restarting failed container=storage-initializer ' 'pod=tensorflow-standard-rest-predictor-5cf5fcc4cc-kntz5_tensorflow-standard(a0ce1273-da25-4d7f-8b65-9b500f160a11)', 'reason': 'CrashLoopBackOff'}}, 'user': {'linux': {'gid': 0, 'supplementalGroups': [0, 1001590000], 'uid': 1001590000}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-l74j8', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'observedGeneration': 1, 'phase': 'Pending', 'podIP': '10.128.0.251', 'podIPs': [{'ip': '10.128.0.251'}], 'qosClass': 'Burstable', 'startTime': '2026-07-09T15:33:14Z'}}Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 361, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 250, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 170, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 536, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1710, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 806, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 630, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 726, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1232, in execute result: FixtureValue = ihook.pytest_fixture_setup( ~~~~~~~~~~~~~~~~~~~~~~~~~~^ fixturedef=self, request=request ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pytest_asyncio/plugin.py", line 924, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper return result.get_result() ~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result raise exc.with_traceback(tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper res = yield ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/setuponly.py", line 36, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1328, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 998, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_runtime/vllm/cpu/cpu_x86/conftest.py", line 101, in cpu_x86_inference_service with create_isvc(**isvc_kwargs) as isvc: ~~~~~~~~~~~^^^^^^^^^^^^^^^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'opt-125m-standard-cpu-predictor-57bf96dbc5-lsm6f': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:33:40Z', 'observedGeneration': 1, 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:33:40Z', 'message': 'containers with incomplete status: [storage-initializer]', 'observedGeneration': 1, 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:33:40Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'observedGeneration': 1, 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:33:40Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'observedGeneration': 1, 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:33:40Z', 'observedGeneration': 1, 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'registry.redhat.io/rhaii-early-access/vllm-cpu-rhel9@sha256:c953143f67160cb6afc3439a788e4f314d31311eacc68c33ff934f69ff023ed7', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/dev/shm', 'name': 'shared-memory'}, {'mountPath': '/tmp', 'name': 'tmp'}, {'mountPath': '/home/vllm', 'name': 'home'}, {'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-qz2th', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}, {'image': 'quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:1e614e46799fd020ac18d95c06a463421154ce86e59ccb3fa6007e66fed9910e', 'imageID': '', 'lastState': {}, 'name': 'kube-rbac-proxy', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/etc/kube-rbac-proxy', 'name': 'kube-rbac-proxy-sar-config', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-qz2th', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '10.0.1.76', 'hostIPs': [{'ip': '10.0.1.76'}], 'initContainerStatuses': [{'allocatedResources': {'cpu': '100m', 'memory': '100Mi'}, 'containerID': 'cri-o://2c0d71b2f5491102ccbaa2e7cf4f0e13e85f53cdae3159b4f7013768f724ca7e', 'image': 'quay.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:166184c520c7821bacbedd05736938b58edc9cc2edc6071bfbc1105a4316aef5', 'imageID': 'quay.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:166184c520c7821bacbedd05736938b58edc9cc2edc6071bfbc1105a4316aef5', 'lastState': {'terminated': {'containerID': 'cri-o://2c0d71b2f5491102ccbaa2e7cf4f0e13e85f53cdae3159b4f7013768f724ca7e', 'exitCode': 1, 'finishedAt': '2026-07-09T15:33:42Z', 'message': '2026-07-09 15:33:42.110 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():17] Initializing, args: ' "(src_uri, dest_path): [('s3://mlflow-e2e/opt-125m/', " "'/mnt/models')]\n" '2026-07-09 15:33:42.110 1 storage.initializer INFO ' '[kserve_storage.py:download():166] Copying contents of ' 's3://mlflow-e2e/opt-125m/ to local\n' '/prod_venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1097: ' 'InsecureRequestWarning: Unverified HTTPS request is being made to ' "host 's3.amazonaws.com'. Adding certificate verification is " 'strongly advised. See: ' 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings\n' ' warnings.warn(\n' '/prod_venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1097: ' 'InsecureRequestWarning: Unverified HTTPS request is being made to ' "host 's3.amazonaws.com'. Adding certificate verification is " 'strongly advised. See: ' 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings\n' ' warnings.warn(\n' '2026-07-09 15:33:42.729 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():28] Storage initialization ' 'failed: Failed to fetch model. No model found in opt-125m/.\n', 'reason': 'Error', 'startedAt': '2026-07-09T15:33:41Z'}}, 'name': 'storage-initializer', 'ready': False, 'resources': {'limits': {'cpu': '1', 'memory': '24Gi'}, 'requests': {'cpu': '100m', 'memory': '100Mi'}}, 'restartCount': 1, 'started': False, 'state': {'waiting': {'message': 'back-off 10s restarting failed container=storage-initializer ' 'pod=opt-125m-standard-cpu-predictor-57bf96dbc5-lsm6f_opt-125m-standard-cpu(e2fa47eb-c29d-449d-9cdd-fbac8ca5fee1)', 'reason': 'CrashLoopBackOff'}}, 'user': {'linux': {'gid': 0, 'supplementalGroups': [0, 1001600000], 'uid': 1001600000}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-qz2th', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'observedGeneration': 1, 'phase': 'Pending', 'podIP': '10.128.0.252', 'podIPs': [{'ip': '10.128.0.252'}], 'qosClass': 'Burstable', 'startTime': '2026-07-09T15:33:40Z'}}Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 361, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 250, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 170, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 536, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1710, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 806, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 630, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 726, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1232, in execute result: FixtureValue = ihook.pytest_fixture_setup( ~~~~~~~~~~~~~~~~~~~~~~~~~~^ fixturedef=self, request=request ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pytest_asyncio/plugin.py", line 924, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper return result.get_result() ~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result raise exc.with_traceback(tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper res = yield ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/setuponly.py", line 36, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1328, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 998, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_runtime/vllm/probes/conftest.py", line 108, in vllm_probes_inference_service with create_isvc(**isvc_kwargs) as isvc: ~~~~~~~~~~~^^^^^^^^^^^^^^^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'opt-125m-probes-predictor-775b68bbfd-v66vt': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:34:04Z', 'observedGeneration': 1, 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:34:04Z', 'message': 'containers with incomplete status: [storage-initializer]', 'observedGeneration': 1, 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:34:04Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'observedGeneration': 1, 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:34:04Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'observedGeneration': 1, 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:34:04Z', 'observedGeneration': 1, 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'registry.redhat.io/rhaii-early-access/vllm-cpu-rhel9@sha256:c953143f67160cb6afc3439a788e4f314d31311eacc68c33ff934f69ff023ed7', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/dev/shm', 'name': 'shared-memory'}, {'mountPath': '/tmp', 'name': 'tmp'}, {'mountPath': '/home/vllm', 'name': 'home'}, {'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-ctx6g', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}, {'image': 'quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:1e614e46799fd020ac18d95c06a463421154ce86e59ccb3fa6007e66fed9910e', 'imageID': '', 'lastState': {}, 'name': 'kube-rbac-proxy', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/etc/kube-rbac-proxy', 'name': 'kube-rbac-proxy-sar-config', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-ctx6g', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '10.0.1.76', 'hostIPs': [{'ip': '10.0.1.76'}], 'initContainerStatuses': [{'allocatedResources': {'cpu': '100m', 'memory': '100Mi'}, 'containerID': 'cri-o://8aac8ff8351ae6bec94e16a535d5f1f9a45ba6b8e07bf0967e05a59077873154', 'image': 'quay.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:166184c520c7821bacbedd05736938b58edc9cc2edc6071bfbc1105a4316aef5', 'imageID': 'quay.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:166184c520c7821bacbedd05736938b58edc9cc2edc6071bfbc1105a4316aef5', 'lastState': {'terminated': {'containerID': 'cri-o://8aac8ff8351ae6bec94e16a535d5f1f9a45ba6b8e07bf0967e05a59077873154', 'exitCode': 1, 'finishedAt': '2026-07-09T15:34:06Z', 'message': '2026-07-09 15:34:06.278 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():17] Initializing, args: ' "(src_uri, dest_path): [('s3://mlflow-e2e/opt-125m/', " "'/mnt/models')]\n" '2026-07-09 15:34:06.278 1 storage.initializer INFO ' '[kserve_storage.py:download():166] Copying contents of ' 's3://mlflow-e2e/opt-125m/ to local\n' '/prod_venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1097: ' 'InsecureRequestWarning: Unverified HTTPS request is being made to ' "host 's3.amazonaws.com'. Adding certificate verification is " 'strongly advised. See: ' 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings\n' ' warnings.warn(\n' '/prod_venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1097: ' 'InsecureRequestWarning: Unverified HTTPS request is being made to ' "host 's3.amazonaws.com'. Adding certificate verification is " 'strongly advised. See: ' 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings\n' ' warnings.warn(\n' '2026-07-09 15:34:06.857 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():28] Storage initialization ' 'failed: Failed to fetch model. No model found in opt-125m/.\n', 'reason': 'Error', 'startedAt': '2026-07-09T15:34:05Z'}}, 'name': 'storage-initializer', 'ready': False, 'resources': {'limits': {'cpu': '1', 'memory': '24Gi'}, 'requests': {'cpu': '100m', 'memory': '100Mi'}}, 'restartCount': 1, 'started': False, 'state': {'waiting': {'message': 'back-off 10s restarting failed container=storage-initializer ' 'pod=opt-125m-probes-predictor-775b68bbfd-v66vt_opt-125m-probes(e94330ce-1916-43d3-932c-185e50c5f02a)', 'reason': 'CrashLoopBackOff'}}, 'user': {'linux': {'gid': 0, 'supplementalGroups': [0, 1001610000], 'uid': 1001610000}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-ctx6g', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'observedGeneration': 1, 'phase': 'Pending', 'podIP': '10.128.0.253', 'podIPs': [{'ip': '10.128.0.253'}], 'qosClass': 'Burstable', 'startTime': '2026-07-09T15:34:04Z'}}Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 361, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 250, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 170, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 536, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1710, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 806, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 630, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 726, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1204, in execute raise exc.with_traceback(exc_tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1328, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 998, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_runtime/vllm/probes/conftest.py", line 108, in vllm_probes_inference_service with create_isvc(**isvc_kwargs) as isvc: ~~~~~~~~~~~^^^^^^^^^^^^^^^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'opt-125m-probes-predictor-775b68bbfd-v66vt': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:34:04Z', 'observedGeneration': 1, 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:34:04Z', 'message': 'containers with incomplete status: [storage-initializer]', 'observedGeneration': 1, 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:34:04Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'observedGeneration': 1, 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:34:04Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'observedGeneration': 1, 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T15:34:04Z', 'observedGeneration': 1, 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'registry.redhat.io/rhaii-early-access/vllm-cpu-rhel9@sha256:c953143f67160cb6afc3439a788e4f314d31311eacc68c33ff934f69ff023ed7', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/dev/shm', 'name': 'shared-memory'}, {'mountPath': '/tmp', 'name': 'tmp'}, {'mountPath': '/home/vllm', 'name': 'home'}, {'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-ctx6g', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}, {'image': 'quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:1e614e46799fd020ac18d95c06a463421154ce86e59ccb3fa6007e66fed9910e', 'imageID': '', 'lastState': {}, 'name': 'kube-rbac-proxy', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/etc/kube-rbac-proxy', 'name': 'kube-rbac-proxy-sar-config', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-ctx6g', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '10.0.1.76', 'hostIPs': [{'ip': '10.0.1.76'}], 'initContainerStatuses': [{'allocatedResources': {'cpu': '100m', 'memory': '100Mi'}, 'containerID': 'cri-o://8aac8ff8351ae6bec94e16a535d5f1f9a45ba6b8e07bf0967e05a59077873154', 'image': 'quay.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:166184c520c7821bacbedd05736938b58edc9cc2edc6071bfbc1105a4316aef5', 'imageID': 'quay.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:166184c520c7821bacbedd05736938b58edc9cc2edc6071bfbc1105a4316aef5', 'lastState': {'terminated': {'containerID': 'cri-o://8aac8ff8351ae6bec94e16a535d5f1f9a45ba6b8e07bf0967e05a59077873154', 'exitCode': 1, 'finishedAt': '2026-07-09T15:34:06Z', 'message': '2026-07-09 15:34:06.278 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():17] Initializing, args: ' "(src_uri, dest_path): [('s3://mlflow-e2e/opt-125m/', " "'/mnt/models')]\n" '2026-07-09 15:34:06.278 1 storage.initializer INFO ' '[kserve_storage.py:download():166] Copying contents of ' 's3://mlflow-e2e/opt-125m/ to local\n' '/prod_venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1097: ' 'InsecureRequestWarning: Unverified HTTPS request is being made to ' "host 's3.amazonaws.com'. Adding certificate verification is " 'strongly advised. See: ' 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings\n' ' warnings.warn(\n' '/prod_venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1097: ' 'InsecureRequestWarning: Unverified HTTPS request is being made to ' "host 's3.amazonaws.com'. Adding certificate verification is " 'strongly advised. See: ' 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings\n' ' warnings.warn(\n' '2026-07-09 15:34:06.857 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():28] Storage initialization ' 'failed: Failed to fetch model. No model found in opt-125m/.\n', 'reason': 'Error', 'startedAt': '2026-07-09T15:34:05Z'}}, 'name': 'storage-initializer', 'ready': False, 'resources': {'limits': {'cpu': '1', 'memory': '24Gi'}, 'requests': {'cpu': '100m', 'memory': '100Mi'}}, 'restartCount': 1, 'started': False, 'state': {'waiting': {'message': 'back-off 10s restarting failed container=storage-initializer ' 'pod=opt-125m-probes-predictor-775b68bbfd-v66vt_opt-125m-probes(e94330ce-1916-43d3-932c-185e50c5f02a)', 'reason': 'CrashLoopBackOff'}}, 'user': {'linux': {'gid': 0, 'supplementalGroups': [0, 1001610000], 'uid': 1001610000}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-ctx6g', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'observedGeneration': 1, 'phase': 'Pending', 'podIP': '10.128.0.253', 'podIPs': [{'ip': '10.128.0.253'}], 'qosClass': 'Burstable', 'startTime': '2026-07-09T15:34:04Z'}}