Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 361, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 250, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 170, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 536, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1710, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 806, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 630, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 726, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1232, in execute result: FixtureValue = ihook.pytest_fixture_setup( ~~~~~~~~~~~~~~~~~~~~~~~~~~^ fixturedef=self, request=request ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pytest_asyncio/plugin.py", line 924, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper return result.get_result() ~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result raise exc.with_traceback(tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper res = yield ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/setuponly.py", line 36, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1328, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 998, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_server/kserve/authentication/conftest.py", line 144, in http_s3_ovms_raw_inference_service with create_isvc( ~~~~~~~~~~~^ client=unprivileged_client, ^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...<9 lines>... external_route=True, ^^^^^^^^^^^^^^^^^^^^ ) as isvc: ^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'http-onnx-predictor-5f65d4b8d7-nvnwq': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:09:43Z', 'observedGeneration': 1, 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:09:42Z', 'message': 'containers with incomplete status: [storage-initializer]', 'observedGeneration': 1, 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:09:42Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'observedGeneration': 1, 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:09:42Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'observedGeneration': 1, 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:09:42Z', 'observedGeneration': 1, 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'quay.io/rhoai/odh-openvino-model-server-rhel9@sha256:8c1b225509bfdc43756195f34a356d1e2ebd58fcd83fda3d37380dd7f5304df8', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-bsmz9', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}, {'image': 'quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:1e614e46799fd020ac18d95c06a463421154ce86e59ccb3fa6007e66fed9910e', 'imageID': '', 'lastState': {}, 'name': 'kube-rbac-proxy', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/etc/kube-rbac-proxy', 'name': 'kube-rbac-proxy-sar-config', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-bsmz9', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '10.0.1.76', 'hostIPs': [{'ip': '10.0.1.76'}], 'initContainerStatuses': [{'allocatedResources': {'cpu': '100m', 'memory': '100Mi'}, 'containerID': 'cri-o://84254ab8bc5868d1939141f4d9a1c7fa61ed2ff23e09a4d75efe2b6f7fbf0e39', 'image': 'quay.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:166184c520c7821bacbedd05736938b58edc9cc2edc6071bfbc1105a4316aef5', 'imageID': 'quay.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:166184c520c7821bacbedd05736938b58edc9cc2edc6071bfbc1105a4316aef5', 'lastState': {'terminated': {'containerID': 'cri-o://84254ab8bc5868d1939141f4d9a1c7fa61ed2ff23e09a4d75efe2b6f7fbf0e39', 'exitCode': 1, 'finishedAt': '2026-07-09T11:09:45Z', 'message': '2026-07-09 11:09:45.295 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():17] Initializing, args: ' "(src_uri, dest_path): [('s3://mlflow-e2e/test-dir/', " "'/mnt/models')]\n" '2026-07-09 11:09:45.295 1 storage.initializer INFO ' '[kserve_storage.py:download():166] Copying contents of ' 's3://mlflow-e2e/test-dir/ to local\n' '2026-07-09 11:09:45.572 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():366] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-07-09 11:09:45.862 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():28] Storage initialization ' 'failed: Failed to fetch model. No model found in test-dir/.\n', 'reason': 'Error', 'startedAt': '2026-07-09T11:09:44Z'}}, 'name': 'storage-initializer', 'ready': False, 'resources': {'limits': {'cpu': '1', 'memory': '24Gi'}, 'requests': {'cpu': '100m', 'memory': '100Mi'}}, 'restartCount': 1, 'started': False, 'state': {'waiting': {'message': 'back-off 10s restarting failed container=storage-initializer ' 'pod=http-onnx-predictor-5f65d4b8d7-nvnwq_test-kserve-raw-token-authentication(e066d29f-da54-4e86-96a1-53d2ecd7fe84)', 'reason': 'CrashLoopBackOff'}}, 'user': {'linux': {'gid': 0, 'supplementalGroups': [0, 1001630000], 'uid': 1001630000}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-bsmz9', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'observedGeneration': 1, 'phase': 'Pending', 'podIP': '10.128.0.29', 'podIPs': [{'ip': '10.128.0.29'}], 'qosClass': 'Burstable', 'startTime': '2026-07-09T11:09:42Z'}}Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 361, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 250, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 170, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 536, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1710, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 806, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 630, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 726, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1204, in execute raise exc.with_traceback(exc_tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1328, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 998, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_server/kserve/authentication/conftest.py", line 144, in http_s3_ovms_raw_inference_service with create_isvc( ~~~~~~~~~~~^ client=unprivileged_client, ^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...<9 lines>... external_route=True, ^^^^^^^^^^^^^^^^^^^^ ) as isvc: ^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'http-onnx-predictor-5f65d4b8d7-nvnwq': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:09:43Z', 'observedGeneration': 1, 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:09:42Z', 'message': 'containers with incomplete status: [storage-initializer]', 'observedGeneration': 1, 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:09:42Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'observedGeneration': 1, 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:09:42Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'observedGeneration': 1, 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:09:42Z', 'observedGeneration': 1, 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'quay.io/rhoai/odh-openvino-model-server-rhel9@sha256:8c1b225509bfdc43756195f34a356d1e2ebd58fcd83fda3d37380dd7f5304df8', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-bsmz9', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}, {'image': 'quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:1e614e46799fd020ac18d95c06a463421154ce86e59ccb3fa6007e66fed9910e', 'imageID': '', 'lastState': {}, 'name': 'kube-rbac-proxy', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/etc/kube-rbac-proxy', 'name': 'kube-rbac-proxy-sar-config', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-bsmz9', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '10.0.1.76', 'hostIPs': [{'ip': '10.0.1.76'}], 'initContainerStatuses': [{'allocatedResources': {'cpu': '100m', 'memory': '100Mi'}, 'containerID': 'cri-o://84254ab8bc5868d1939141f4d9a1c7fa61ed2ff23e09a4d75efe2b6f7fbf0e39', 'image': 'quay.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:166184c520c7821bacbedd05736938b58edc9cc2edc6071bfbc1105a4316aef5', 'imageID': 'quay.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:166184c520c7821bacbedd05736938b58edc9cc2edc6071bfbc1105a4316aef5', 'lastState': {'terminated': {'containerID': 'cri-o://84254ab8bc5868d1939141f4d9a1c7fa61ed2ff23e09a4d75efe2b6f7fbf0e39', 'exitCode': 1, 'finishedAt': '2026-07-09T11:09:45Z', 'message': '2026-07-09 11:09:45.295 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():17] Initializing, args: ' "(src_uri, dest_path): [('s3://mlflow-e2e/test-dir/', " "'/mnt/models')]\n" '2026-07-09 11:09:45.295 1 storage.initializer INFO ' '[kserve_storage.py:download():166] Copying contents of ' 's3://mlflow-e2e/test-dir/ to local\n' '2026-07-09 11:09:45.572 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():366] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-07-09 11:09:45.862 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():28] Storage initialization ' 'failed: Failed to fetch model. No model found in test-dir/.\n', 'reason': 'Error', 'startedAt': '2026-07-09T11:09:44Z'}}, 'name': 'storage-initializer', 'ready': False, 'resources': {'limits': {'cpu': '1', 'memory': '24Gi'}, 'requests': {'cpu': '100m', 'memory': '100Mi'}}, 'restartCount': 1, 'started': False, 'state': {'waiting': {'message': 'back-off 10s restarting failed container=storage-initializer ' 'pod=http-onnx-predictor-5f65d4b8d7-nvnwq_test-kserve-raw-token-authentication(e066d29f-da54-4e86-96a1-53d2ecd7fe84)', 'reason': 'CrashLoopBackOff'}}, 'user': {'linux': {'gid': 0, 'supplementalGroups': [0, 1001630000], 'uid': 1001630000}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-bsmz9', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'observedGeneration': 1, 'phase': 'Pending', 'podIP': '10.128.0.29', 'podIPs': [{'ip': '10.128.0.29'}], 'qosClass': 'Burstable', 'startTime': '2026-07-09T11:09:42Z'}}Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 361, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 250, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 170, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 536, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1710, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 806, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 630, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 726, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1204, in execute raise exc.with_traceback(exc_tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1328, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 998, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_server/kserve/authentication/conftest.py", line 144, in http_s3_ovms_raw_inference_service with create_isvc( ~~~~~~~~~~~^ client=unprivileged_client, ^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...<9 lines>... external_route=True, ^^^^^^^^^^^^^^^^^^^^ ) as isvc: ^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'http-onnx-predictor-5f65d4b8d7-nvnwq': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:09:43Z', 'observedGeneration': 1, 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:09:42Z', 'message': 'containers with incomplete status: [storage-initializer]', 'observedGeneration': 1, 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:09:42Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'observedGeneration': 1, 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:09:42Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'observedGeneration': 1, 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:09:42Z', 'observedGeneration': 1, 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'quay.io/rhoai/odh-openvino-model-server-rhel9@sha256:8c1b225509bfdc43756195f34a356d1e2ebd58fcd83fda3d37380dd7f5304df8', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-bsmz9', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}, {'image': 'quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:1e614e46799fd020ac18d95c06a463421154ce86e59ccb3fa6007e66fed9910e', 'imageID': '', 'lastState': {}, 'name': 'kube-rbac-proxy', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/etc/kube-rbac-proxy', 'name': 'kube-rbac-proxy-sar-config', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-bsmz9', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '10.0.1.76', 'hostIPs': [{'ip': '10.0.1.76'}], 'initContainerStatuses': [{'allocatedResources': {'cpu': '100m', 'memory': '100Mi'}, 'containerID': 'cri-o://84254ab8bc5868d1939141f4d9a1c7fa61ed2ff23e09a4d75efe2b6f7fbf0e39', 'image': 'quay.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:166184c520c7821bacbedd05736938b58edc9cc2edc6071bfbc1105a4316aef5', 'imageID': 'quay.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:166184c520c7821bacbedd05736938b58edc9cc2edc6071bfbc1105a4316aef5', 'lastState': {'terminated': {'containerID': 'cri-o://84254ab8bc5868d1939141f4d9a1c7fa61ed2ff23e09a4d75efe2b6f7fbf0e39', 'exitCode': 1, 'finishedAt': '2026-07-09T11:09:45Z', 'message': '2026-07-09 11:09:45.295 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():17] Initializing, args: ' "(src_uri, dest_path): [('s3://mlflow-e2e/test-dir/', " "'/mnt/models')]\n" '2026-07-09 11:09:45.295 1 storage.initializer INFO ' '[kserve_storage.py:download():166] Copying contents of ' 's3://mlflow-e2e/test-dir/ to local\n' '2026-07-09 11:09:45.572 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():366] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-07-09 11:09:45.862 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():28] Storage initialization ' 'failed: Failed to fetch model. No model found in test-dir/.\n', 'reason': 'Error', 'startedAt': '2026-07-09T11:09:44Z'}}, 'name': 'storage-initializer', 'ready': False, 'resources': {'limits': {'cpu': '1', 'memory': '24Gi'}, 'requests': {'cpu': '100m', 'memory': '100Mi'}}, 'restartCount': 1, 'started': False, 'state': {'waiting': {'message': 'back-off 10s restarting failed container=storage-initializer ' 'pod=http-onnx-predictor-5f65d4b8d7-nvnwq_test-kserve-raw-token-authentication(e066d29f-da54-4e86-96a1-53d2ecd7fe84)', 'reason': 'CrashLoopBackOff'}}, 'user': {'linux': {'gid': 0, 'supplementalGroups': [0, 1001630000], 'uid': 1001630000}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-bsmz9', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'observedGeneration': 1, 'phase': 'Pending', 'podIP': '10.128.0.29', 'podIPs': [{'ip': '10.128.0.29'}], 'qosClass': 'Burstable', 'startTime': '2026-07-09T11:09:42Z'}}Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 361, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 250, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 170, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 536, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1710, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 806, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 630, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 726, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1232, in execute result: FixtureValue = ihook.pytest_fixture_setup( ~~~~~~~~~~~~~~~~~~~~~~~~~~^ fixturedef=self, request=request ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pytest_asyncio/plugin.py", line 924, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper return result.get_result() ~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result raise exc.with_traceback(tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper res = yield ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/setuponly.py", line 36, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1328, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 998, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_server/kserve/authentication/conftest.py", line 172, in http_s3_ovms_raw_inference_service_2 with create_isvc( ~~~~~~~~~~~^ client=unprivileged_client, ^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...<9 lines>... external_route=True, ^^^^^^^^^^^^^^^^^^^^ ) as isvc: ^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'http-onnx-2-predictor-7fc87c5c6c-8htgx': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:09:56Z', 'observedGeneration': 1, 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:09:54Z', 'message': 'containers with incomplete status: [storage-initializer]', 'observedGeneration': 1, 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:09:54Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'observedGeneration': 1, 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:09:54Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'observedGeneration': 1, 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:09:54Z', 'observedGeneration': 1, 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'quay.io/rhoai/odh-openvino-model-server-rhel9@sha256:8c1b225509bfdc43756195f34a356d1e2ebd58fcd83fda3d37380dd7f5304df8', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-pcwqj', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}, {'image': 'quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:1e614e46799fd020ac18d95c06a463421154ce86e59ccb3fa6007e66fed9910e', 'imageID': '', 'lastState': {}, 'name': 'kube-rbac-proxy', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/etc/kube-rbac-proxy', 'name': 'kube-rbac-proxy-sar-config', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-pcwqj', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '10.0.1.76', 'hostIPs': [{'ip': '10.0.1.76'}], 'initContainerStatuses': [{'allocatedResources': {'cpu': '100m', 'memory': '100Mi'}, 'containerID': 'cri-o://0e6926e8a53e3d547f688d75f0bad2d852377f05aa16b7fff3583da66e70a4e7', 'image': 'quay.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:166184c520c7821bacbedd05736938b58edc9cc2edc6071bfbc1105a4316aef5', 'imageID': 'quay.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:166184c520c7821bacbedd05736938b58edc9cc2edc6071bfbc1105a4316aef5', 'lastState': {'terminated': {'containerID': 'cri-o://0e6926e8a53e3d547f688d75f0bad2d852377f05aa16b7fff3583da66e70a4e7', 'exitCode': 1, 'finishedAt': '2026-07-09T11:09:58Z', 'message': '2026-07-09 11:09:57.383 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():17] Initializing, args: ' "(src_uri, dest_path): [('s3://mlflow-e2e/test-dir/', " "'/mnt/models')]\n" '2026-07-09 11:09:57.384 1 storage.initializer INFO ' '[kserve_storage.py:download():166] Copying contents of ' 's3://mlflow-e2e/test-dir/ to local\n' '2026-07-09 11:09:57.666 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():366] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-07-09 11:09:57.974 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():28] Storage initialization ' 'failed: Failed to fetch model. No model found in test-dir/.\n', 'reason': 'Error', 'startedAt': '2026-07-09T11:09:57Z'}}, 'name': 'storage-initializer', 'ready': False, 'resources': {'limits': {'cpu': '1', 'memory': '24Gi'}, 'requests': {'cpu': '100m', 'memory': '100Mi'}}, 'restartCount': 1, 'started': False, 'state': {'waiting': {'message': 'back-off 10s restarting failed container=storage-initializer ' 'pod=http-onnx-2-predictor-7fc87c5c6c-8htgx_test-kserve-raw-token-authentication(1f4d989d-47d1-480f-a1ef-9d1b1b1ed853)', 'reason': 'CrashLoopBackOff'}}, 'user': {'linux': {'gid': 0, 'supplementalGroups': [0, 1001630000], 'uid': 1001630000}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-pcwqj', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'observedGeneration': 1, 'phase': 'Pending', 'podIP': '10.128.0.30', 'podIPs': [{'ip': '10.128.0.30'}], 'qosClass': 'Burstable', 'startTime': '2026-07-09T11:09:54Z'}}Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 361, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 250, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 170, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 536, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1710, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 806, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 630, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 726, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1232, in execute result: FixtureValue = ihook.pytest_fixture_setup( ~~~~~~~~~~~~~~~~~~~~~~~~~~^ fixturedef=self, request=request ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pytest_asyncio/plugin.py", line 924, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper return result.get_result() ~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result raise exc.with_traceback(tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper res = yield ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/setuponly.py", line 36, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1328, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 998, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_server/kserve/authentication/conftest.py", line 214, in unprivileged_s3_ovms_raw_inference_service with create_isvc( ~~~~~~~~~~~^ client=unprivileged_client, ^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...<6 lines>... storage_path=request.param["model-dir"], ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) as isvc: ^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'http-onnx-raw-predictor-6d45f84b54-t42wx': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:10:21Z', 'observedGeneration': 1, 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:10:19Z', 'message': 'containers with incomplete status: [storage-initializer]', 'observedGeneration': 1, 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:10:19Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'observedGeneration': 1, 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:10:19Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'observedGeneration': 1, 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-07-09T11:10:19Z', 'observedGeneration': 1, 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'quay.io/rhoai/odh-openvino-model-server-rhel9@sha256:8c1b225509bfdc43756195f34a356d1e2ebd58fcd83fda3d37380dd7f5304df8', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-8hglz', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}, {'image': 'quay.io/rhoai/odh-kube-rbac-proxy-rhel9@sha256:1e614e46799fd020ac18d95c06a463421154ce86e59ccb3fa6007e66fed9910e', 'imageID': '', 'lastState': {}, 'name': 'kube-rbac-proxy', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/etc/kube-rbac-proxy', 'name': 'kube-rbac-proxy-sar-config', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-8hglz', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '10.0.1.76', 'hostIPs': [{'ip': '10.0.1.76'}], 'initContainerStatuses': [{'allocatedResources': {'cpu': '100m', 'memory': '100Mi'}, 'containerID': 'cri-o://f65464090c716aabf431b3025dfe66b2211b81f8481dfb0712d49e76c2e54e8a', 'image': 'quay.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:166184c520c7821bacbedd05736938b58edc9cc2edc6071bfbc1105a4316aef5', 'imageID': 'quay.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:166184c520c7821bacbedd05736938b58edc9cc2edc6071bfbc1105a4316aef5', 'lastState': {'terminated': {'containerID': 'cri-o://f65464090c716aabf431b3025dfe66b2211b81f8481dfb0712d49e76c2e54e8a', 'exitCode': 1, 'finishedAt': '2026-07-09T11:10:23Z', 'message': '2026-07-09 11:10:22.562 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():17] Initializing, args: ' "(src_uri, dest_path): [('s3://mlflow-e2e/test-dir/', " "'/mnt/models')]\n" '2026-07-09 11:10:22.562 1 storage.initializer INFO ' '[kserve_storage.py:download():166] Copying contents of ' 's3://mlflow-e2e/test-dir/ to local\n' '2026-07-09 11:10:22.845 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():366] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-07-09 11:10:23.150 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():28] Storage initialization ' 'failed: Failed to fetch model. No model found in test-dir/.\n', 'reason': 'Error', 'startedAt': '2026-07-09T11:10:22Z'}}, 'name': 'storage-initializer', 'ready': False, 'resources': {'limits': {'cpu': '1', 'memory': '24Gi'}, 'requests': {'cpu': '100m', 'memory': '100Mi'}}, 'restartCount': 1, 'started': False, 'state': {'waiting': {'message': 'back-off 10s restarting failed container=storage-initializer ' 'pod=http-onnx-raw-predictor-6d45f84b54-t42wx_test-non-admin-raw(55d8217c-e595-4e37-9c1e-586e928f132c)', 'reason': 'CrashLoopBackOff'}}, 'user': {'linux': {'gid': 0, 'supplementalGroups': [0, 1001650000], 'uid': 1001650000}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-8hglz', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'observedGeneration': 1, 'phase': 'Pending', 'podIP': '10.128.0.32', 'podIPs': [{'ip': '10.128.0.32'}], 'qosClass': 'Burstable', 'startTime': '2026-07-09T11:10:19Z'}}