Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 353, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 245, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 165, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 523, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1723, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 707, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 539, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 627, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1110, in execute result: FixtureValue = ihook.pytest_fixture_setup( ~~~~~~~~~~~~~~~~~~~~~~~~~~^ fixturedef=self, request=request ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pytest_asyncio/plugin.py", line 728, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper return result.get_result() ~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result raise exc.with_traceback(tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper res = yield ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/setuponly.py", line 36, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1202, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 908, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_server/kserve/authentication/conftest.py", line 144, in http_s3_ovms_raw_inference_service with create_isvc( ~~~~~~~~~~~^ client=unprivileged_client, ^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...<9 lines>... external_route=True, ^^^^^^^^^^^^^^^^^^^^ ) as isvc: ^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'http-onnx-predictor-86778bd869-zhmcb': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:25:26Z', 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:25:25Z', 'message': 'containers with incomplete status: [storage-initializer]', 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:25:25Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:25:25Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:25:25Z', 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'registry.redhat.io/rhoai/odh-openvino-model-server-rhel9@sha256:1ab58519c50e2c3a9ebf0fee6d0708b1b5a0ae972aefcc722d87b2f62239a033', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-zn8d2', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}, {'image': 'registry.redhat.io/rhoai/odh-kube-auth-proxy-rhel9@sha256:07ee0914946d9d6b2c7c7367be5b6bc13ddf0dddd93e896fc50aa617d761dc3a', 'imageID': '', 'lastState': {}, 'name': 'kube-rbac-proxy', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/etc/kube-rbac-proxy', 'name': 'http-onnx-kube-rbac-proxy-sar-config', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-zn8d2', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '192.169.1.109', 'hostIPs': [{'ip': '192.169.1.109'}], 'initContainerStatuses': [{'allocatedResources': {'cpu': '100m', 'memory': '100Mi'}, 'containerID': 'cri-o://98d831dbb1a168fe7e547543259e44e7da2717050e475e8d2e4a6d27f1607940', 'image': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:b73f3818b1c3b6f57d9cac6b975552c5888bf9c0d37df6c50938fd4988d2012b', 'imageID': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:9bb77298f77598b5a61a2aeaad625500a0e17b64e8da4b7a1988efb99bda2660', 'lastState': {'terminated': {'containerID': 'cri-o://825c1f60f01adaf3cd7b6548b28a201b41cda9235a78d27ed6415bf3e77af8cd', 'exitCode': 1, 'finishedAt': '2026-06-12T14:25:32Z', 'message': '2026-06-12 14:25:26.459 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' "(src_uri, dest_path): [('s3://ods-ci-s3/test-dir/', " "'/mnt/models')]\n" '2026-06-12 14:25:26.460 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://ods-ci-s3/test-dir/ to local\n' '2026-06-12 14:25:26.774 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():361] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-06-12 14:25:31.955 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():24] Unexpected error during ' 'storage initialization: SSL validation failed for ' 'https://s3.us-east-1.amazonaws.com/ods-ci-s3 [SSL: ' 'CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to ' 'get local issuer certificate (_ssl.c:1004)\n', 'reason': 'Error', 'startedAt': '2026-06-12T14:25:26Z'}}, 'name': 'storage-initializer', 'ready': False, 'resources': {'limits': {'cpu': '1', 'memory': '24Gi'}, 'requests': {'cpu': '100m', 'memory': '100Mi'}}, 'restartCount': 1, 'started': False, 'state': {'terminated': {'containerID': 'cri-o://98d831dbb1a168fe7e547543259e44e7da2717050e475e8d2e4a6d27f1607940', 'exitCode': 1, 'finishedAt': '2026-06-12T14:25:35Z', 'message': '2026-06-12 14:25:33.392 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' "(src_uri, dest_path): [('s3://ods-ci-s3/test-dir/', " "'/mnt/models')]\n" '2026-06-12 14:25:33.392 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://ods-ci-s3/test-dir/ to local\n' '2026-06-12 14:25:33.675 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():361] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-06-12 14:25:35.779 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():24] Unexpected error during ' 'storage initialization: SSL validation failed for ' 'https://s3.us-east-1.amazonaws.com/ods-ci-s3 [SSL: ' 'CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to ' 'get local issuer certificate (_ssl.c:1004)\n', 'reason': 'Error', 'startedAt': '2026-06-12T14:25:33Z'}}, 'user': {'linux': {'gid': 0, 'supplementalGroups': [0, 1001680000], 'uid': 1001680000}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-zn8d2', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'phase': 'Pending', 'podIP': '10.131.1.213', 'podIPs': [{'ip': '10.131.1.213'}], 'qosClass': 'Burstable', 'startTime': '2026-06-12T14:25:25Z'}}Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 353, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 245, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 165, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 523, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1723, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 707, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 539, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 627, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1091, in execute raise exc.with_traceback(exc_tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1202, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 908, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_server/kserve/authentication/conftest.py", line 144, in http_s3_ovms_raw_inference_service with create_isvc( ~~~~~~~~~~~^ client=unprivileged_client, ^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...<9 lines>... external_route=True, ^^^^^^^^^^^^^^^^^^^^ ) as isvc: ^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'http-onnx-predictor-86778bd869-zhmcb': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:25:26Z', 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:25:25Z', 'message': 'containers with incomplete status: [storage-initializer]', 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:25:25Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:25:25Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:25:25Z', 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'registry.redhat.io/rhoai/odh-openvino-model-server-rhel9@sha256:1ab58519c50e2c3a9ebf0fee6d0708b1b5a0ae972aefcc722d87b2f62239a033', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-zn8d2', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}, {'image': 'registry.redhat.io/rhoai/odh-kube-auth-proxy-rhel9@sha256:07ee0914946d9d6b2c7c7367be5b6bc13ddf0dddd93e896fc50aa617d761dc3a', 'imageID': '', 'lastState': {}, 'name': 'kube-rbac-proxy', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/etc/kube-rbac-proxy', 'name': 'http-onnx-kube-rbac-proxy-sar-config', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-zn8d2', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '192.169.1.109', 'hostIPs': [{'ip': '192.169.1.109'}], 'initContainerStatuses': [{'allocatedResources': {'cpu': '100m', 'memory': '100Mi'}, 'containerID': 'cri-o://98d831dbb1a168fe7e547543259e44e7da2717050e475e8d2e4a6d27f1607940', 'image': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:b73f3818b1c3b6f57d9cac6b975552c5888bf9c0d37df6c50938fd4988d2012b', 'imageID': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:9bb77298f77598b5a61a2aeaad625500a0e17b64e8da4b7a1988efb99bda2660', 'lastState': {'terminated': {'containerID': 'cri-o://825c1f60f01adaf3cd7b6548b28a201b41cda9235a78d27ed6415bf3e77af8cd', 'exitCode': 1, 'finishedAt': '2026-06-12T14:25:32Z', 'message': '2026-06-12 14:25:26.459 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' "(src_uri, dest_path): [('s3://ods-ci-s3/test-dir/', " "'/mnt/models')]\n" '2026-06-12 14:25:26.460 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://ods-ci-s3/test-dir/ to local\n' '2026-06-12 14:25:26.774 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():361] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-06-12 14:25:31.955 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():24] Unexpected error during ' 'storage initialization: SSL validation failed for ' 'https://s3.us-east-1.amazonaws.com/ods-ci-s3 [SSL: ' 'CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to ' 'get local issuer certificate (_ssl.c:1004)\n', 'reason': 'Error', 'startedAt': '2026-06-12T14:25:26Z'}}, 'name': 'storage-initializer', 'ready': False, 'resources': {'limits': {'cpu': '1', 'memory': '24Gi'}, 'requests': {'cpu': '100m', 'memory': '100Mi'}}, 'restartCount': 1, 'started': False, 'state': {'terminated': {'containerID': 'cri-o://98d831dbb1a168fe7e547543259e44e7da2717050e475e8d2e4a6d27f1607940', 'exitCode': 1, 'finishedAt': '2026-06-12T14:25:35Z', 'message': '2026-06-12 14:25:33.392 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' "(src_uri, dest_path): [('s3://ods-ci-s3/test-dir/', " "'/mnt/models')]\n" '2026-06-12 14:25:33.392 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://ods-ci-s3/test-dir/ to local\n' '2026-06-12 14:25:33.675 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():361] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-06-12 14:25:35.779 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():24] Unexpected error during ' 'storage initialization: SSL validation failed for ' 'https://s3.us-east-1.amazonaws.com/ods-ci-s3 [SSL: ' 'CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to ' 'get local issuer certificate (_ssl.c:1004)\n', 'reason': 'Error', 'startedAt': '2026-06-12T14:25:33Z'}}, 'user': {'linux': {'gid': 0, 'supplementalGroups': [0, 1001680000], 'uid': 1001680000}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-zn8d2', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'phase': 'Pending', 'podIP': '10.131.1.213', 'podIPs': [{'ip': '10.131.1.213'}], 'qosClass': 'Burstable', 'startTime': '2026-06-12T14:25:25Z'}}Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 353, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 245, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 165, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 523, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1723, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 707, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 539, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 627, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1091, in execute raise exc.with_traceback(exc_tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1202, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 908, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_server/kserve/authentication/conftest.py", line 144, in http_s3_ovms_raw_inference_service with create_isvc( ~~~~~~~~~~~^ client=unprivileged_client, ^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...<9 lines>... external_route=True, ^^^^^^^^^^^^^^^^^^^^ ) as isvc: ^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'http-onnx-predictor-86778bd869-zhmcb': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:25:26Z', 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:25:25Z', 'message': 'containers with incomplete status: [storage-initializer]', 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:25:25Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:25:25Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:25:25Z', 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'registry.redhat.io/rhoai/odh-openvino-model-server-rhel9@sha256:1ab58519c50e2c3a9ebf0fee6d0708b1b5a0ae972aefcc722d87b2f62239a033', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-zn8d2', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}, {'image': 'registry.redhat.io/rhoai/odh-kube-auth-proxy-rhel9@sha256:07ee0914946d9d6b2c7c7367be5b6bc13ddf0dddd93e896fc50aa617d761dc3a', 'imageID': '', 'lastState': {}, 'name': 'kube-rbac-proxy', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/etc/kube-rbac-proxy', 'name': 'http-onnx-kube-rbac-proxy-sar-config', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-zn8d2', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '192.169.1.109', 'hostIPs': [{'ip': '192.169.1.109'}], 'initContainerStatuses': [{'allocatedResources': {'cpu': '100m', 'memory': '100Mi'}, 'containerID': 'cri-o://98d831dbb1a168fe7e547543259e44e7da2717050e475e8d2e4a6d27f1607940', 'image': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:b73f3818b1c3b6f57d9cac6b975552c5888bf9c0d37df6c50938fd4988d2012b', 'imageID': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:9bb77298f77598b5a61a2aeaad625500a0e17b64e8da4b7a1988efb99bda2660', 'lastState': {'terminated': {'containerID': 'cri-o://825c1f60f01adaf3cd7b6548b28a201b41cda9235a78d27ed6415bf3e77af8cd', 'exitCode': 1, 'finishedAt': '2026-06-12T14:25:32Z', 'message': '2026-06-12 14:25:26.459 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' "(src_uri, dest_path): [('s3://ods-ci-s3/test-dir/', " "'/mnt/models')]\n" '2026-06-12 14:25:26.460 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://ods-ci-s3/test-dir/ to local\n' '2026-06-12 14:25:26.774 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():361] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-06-12 14:25:31.955 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():24] Unexpected error during ' 'storage initialization: SSL validation failed for ' 'https://s3.us-east-1.amazonaws.com/ods-ci-s3 [SSL: ' 'CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to ' 'get local issuer certificate (_ssl.c:1004)\n', 'reason': 'Error', 'startedAt': '2026-06-12T14:25:26Z'}}, 'name': 'storage-initializer', 'ready': False, 'resources': {'limits': {'cpu': '1', 'memory': '24Gi'}, 'requests': {'cpu': '100m', 'memory': '100Mi'}}, 'restartCount': 1, 'started': False, 'state': {'terminated': {'containerID': 'cri-o://98d831dbb1a168fe7e547543259e44e7da2717050e475e8d2e4a6d27f1607940', 'exitCode': 1, 'finishedAt': '2026-06-12T14:25:35Z', 'message': '2026-06-12 14:25:33.392 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' "(src_uri, dest_path): [('s3://ods-ci-s3/test-dir/', " "'/mnt/models')]\n" '2026-06-12 14:25:33.392 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://ods-ci-s3/test-dir/ to local\n' '2026-06-12 14:25:33.675 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():361] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-06-12 14:25:35.779 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():24] Unexpected error during ' 'storage initialization: SSL validation failed for ' 'https://s3.us-east-1.amazonaws.com/ods-ci-s3 [SSL: ' 'CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to ' 'get local issuer certificate (_ssl.c:1004)\n', 'reason': 'Error', 'startedAt': '2026-06-12T14:25:33Z'}}, 'user': {'linux': {'gid': 0, 'supplementalGroups': [0, 1001680000], 'uid': 1001680000}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-zn8d2', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'phase': 'Pending', 'podIP': '10.131.1.213', 'podIPs': [{'ip': '10.131.1.213'}], 'qosClass': 'Burstable', 'startTime': '2026-06-12T14:25:25Z'}}Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 353, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 245, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 165, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 523, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1723, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 707, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 539, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 627, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1110, in execute result: FixtureValue = ihook.pytest_fixture_setup( ~~~~~~~~~~~~~~~~~~~~~~~~~~^ fixturedef=self, request=request ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pytest_asyncio/plugin.py", line 728, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper return result.get_result() ~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result raise exc.with_traceback(tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper res = yield ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/setuponly.py", line 36, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1202, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 908, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_server/kserve/authentication/conftest.py", line 172, in http_s3_ovms_raw_inference_service_2 with create_isvc( ~~~~~~~~~~~^ client=unprivileged_client, ^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...<9 lines>... external_route=True, ^^^^^^^^^^^^^^^^^^^^ ) as isvc: ^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'http-onnx-2-predictor-84b9789455-m4sjq': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:25:39Z', 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:25:37Z', 'message': 'containers with incomplete status: [storage-initializer]', 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:25:37Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:25:37Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:25:37Z', 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'registry.redhat.io/rhoai/odh-openvino-model-server-rhel9@sha256:1ab58519c50e2c3a9ebf0fee6d0708b1b5a0ae972aefcc722d87b2f62239a033', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-n6qdh', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}, {'image': 'registry.redhat.io/rhoai/odh-kube-auth-proxy-rhel9@sha256:07ee0914946d9d6b2c7c7367be5b6bc13ddf0dddd93e896fc50aa617d761dc3a', 'imageID': '', 'lastState': {}, 'name': 'kube-rbac-proxy', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/etc/kube-rbac-proxy', 'name': 'http-onnx-2-kube-rbac-proxy-sar-config', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-n6qdh', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '192.169.0.196', 'hostIPs': [{'ip': '192.169.0.196'}], 'initContainerStatuses': [{'allocatedResources': {'cpu': '100m', 'memory': '100Mi'}, 'containerID': 'cri-o://c558dfd24a28cf216d76f86cb91b867e724f2ea416956804c74bddab370ed6c3', 'image': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:b73f3818b1c3b6f57d9cac6b975552c5888bf9c0d37df6c50938fd4988d2012b', 'imageID': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:9bb77298f77598b5a61a2aeaad625500a0e17b64e8da4b7a1988efb99bda2660', 'lastState': {'terminated': {'containerID': 'cri-o://bcc9276e3d96e86a41d6cc1ed07c076af08771e4838912509da76f4642d2894d', 'exitCode': 1, 'finishedAt': '2026-06-12T14:25:43Z', 'message': '2026-06-12 14:25:38.796 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' "(src_uri, dest_path): [('s3://ods-ci-s3/test-dir/', " "'/mnt/models')]\n" '2026-06-12 14:25:38.796 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://ods-ci-s3/test-dir/ to local\n' '2026-06-12 14:25:39.061 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():361] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-06-12 14:25:43.735 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():24] Unexpected error during ' 'storage initialization: SSL validation failed for ' 'https://s3.us-east-1.amazonaws.com/ods-ci-s3 [SSL: ' 'CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to ' 'get local issuer certificate (_ssl.c:1004)\n', 'reason': 'Error', 'startedAt': '2026-06-12T14:25:38Z'}}, 'name': 'storage-initializer', 'ready': False, 'resources': {'limits': {'cpu': '1', 'memory': '24Gi'}, 'requests': {'cpu': '100m', 'memory': '100Mi'}}, 'restartCount': 1, 'started': False, 'state': {'terminated': {'containerID': 'cri-o://c558dfd24a28cf216d76f86cb91b867e724f2ea416956804c74bddab370ed6c3', 'exitCode': 1, 'finishedAt': '2026-06-12T14:25:50Z', 'message': '2026-06-12 14:25:44.523 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' "(src_uri, dest_path): [('s3://ods-ci-s3/test-dir/', " "'/mnt/models')]\n" '2026-06-12 14:25:44.523 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://ods-ci-s3/test-dir/ to local\n' '2026-06-12 14:25:44.802 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():361] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-06-12 14:25:50.413 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():24] Unexpected error during ' 'storage initialization: SSL validation failed for ' 'https://s3.us-east-1.amazonaws.com/ods-ci-s3 [SSL: ' 'CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to ' 'get local issuer certificate (_ssl.c:1004)\n', 'reason': 'Error', 'startedAt': '2026-06-12T14:25:44Z'}}, 'user': {'linux': {'gid': 0, 'supplementalGroups': [0, 1001680000], 'uid': 1001680000}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-n6qdh', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'phase': 'Pending', 'podIP': '10.128.3.76', 'podIPs': [{'ip': '10.128.3.76'}], 'qosClass': 'Burstable', 'startTime': '2026-06-12T14:25:37Z'}}Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 353, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 245, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 165, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 523, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1723, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 707, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 539, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 627, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1110, in execute result: FixtureValue = ihook.pytest_fixture_setup( ~~~~~~~~~~~~~~~~~~~~~~~~~~^ fixturedef=self, request=request ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pytest_asyncio/plugin.py", line 728, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper return result.get_result() ~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result raise exc.with_traceback(tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper res = yield ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/setuponly.py", line 36, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1202, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 908, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_server/kserve/authentication/conftest.py", line 214, in unprivileged_s3_ovms_raw_inference_service with create_isvc( ~~~~~~~~~~~^ client=unprivileged_client, ^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...<6 lines>... storage_path=request.param["model-dir"], ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) as isvc: ^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'http-onnx-raw-predictor-7d584db466-xld6p': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:26:10Z', 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:26:08Z', 'message': 'containers with incomplete status: [storage-initializer]', 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:26:08Z', 'message': 'containers with unready status: [kserve-container]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:26:08Z', 'message': 'containers with unready status: [kserve-container]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:26:08Z', 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'registry.redhat.io/rhoai/odh-openvino-model-server-rhel9@sha256:1ab58519c50e2c3a9ebf0fee6d0708b1b5a0ae972aefcc722d87b2f62239a033', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '192.169.1.109', 'hostIPs': [{'ip': '192.169.1.109'}], 'initContainerStatuses': [{'allocatedResources': {'cpu': '100m', 'memory': '100Mi'}, 'containerID': 'cri-o://36e68a0a1df7750460f9c69600e1e449c7e424a40326cd9e3f3036c2a0253cea', 'image': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:b73f3818b1c3b6f57d9cac6b975552c5888bf9c0d37df6c50938fd4988d2012b', 'imageID': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:9bb77298f77598b5a61a2aeaad625500a0e17b64e8da4b7a1988efb99bda2660', 'lastState': {'terminated': {'containerID': 'cri-o://564e40928c8995591f9b94c6f5d03a74322f924a17e21f97cf0ebcfaebff207f', 'exitCode': 1, 'finishedAt': '2026-06-12T14:26:16Z', 'message': '2026-06-12 14:26:10.047 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' "(src_uri, dest_path): [('s3://ods-ci-s3/test-dir/', " "'/mnt/models')]\n" '2026-06-12 14:26:10.048 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://ods-ci-s3/test-dir/ to local\n' '2026-06-12 14:26:10.328 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():361] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-06-12 14:26:15.971 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():24] Unexpected error during ' 'storage initialization: SSL validation failed for ' 'https://s3.us-east-1.amazonaws.com/ods-ci-s3 [SSL: ' 'CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to ' 'get local issuer certificate (_ssl.c:1004)\n', 'reason': 'Error', 'startedAt': '2026-06-12T14:26:09Z'}}, 'name': 'storage-initializer', 'ready': False, 'resources': {'limits': {'cpu': '1', 'memory': '24Gi'}, 'requests': {'cpu': '100m', 'memory': '100Mi'}}, 'restartCount': 1, 'started': False, 'state': {'terminated': {'containerID': 'cri-o://36e68a0a1df7750460f9c69600e1e449c7e424a40326cd9e3f3036c2a0253cea', 'exitCode': 1, 'finishedAt': '2026-06-12T14:26:20Z', 'message': '2026-06-12 14:26:16.596 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' "(src_uri, dest_path): [('s3://ods-ci-s3/test-dir/', " "'/mnt/models')]\n" '2026-06-12 14:26:16.596 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://ods-ci-s3/test-dir/ to local\n' '2026-06-12 14:26:16.894 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():361] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-06-12 14:26:20.369 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():24] Unexpected error during ' 'storage initialization: SSL validation failed for ' 'https://s3.us-east-1.amazonaws.com/ods-ci-s3 [SSL: ' 'CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to ' 'get local issuer certificate (_ssl.c:1004)\n', 'reason': 'Error', 'startedAt': '2026-06-12T14:26:16Z'}}, 'user': {'linux': {'gid': 0, 'supplementalGroups': [0, 1001690000], 'uid': 1001690000}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'phase': 'Pending', 'podIP': '10.131.1.214', 'podIPs': [{'ip': '10.131.1.214'}], 'qosClass': 'Burstable', 'startTime': '2026-06-12T14:26:08Z'}}