Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 353, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 245, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 165, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 523, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1723, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 707, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 539, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 627, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1110, in execute result: FixtureValue = ihook.pytest_fixture_setup( ~~~~~~~~~~~~~~~~~~~~~~~~~~^ fixturedef=self, request=request ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pytest_asyncio/plugin.py", line 728, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper return result.get_result() ~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result raise exc.with_traceback(tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper res = yield ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/setuponly.py", line 36, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1202, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 908, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_server/kserve/authentication/conftest.py", line 144, in http_s3_ovms_raw_inference_service with create_isvc( ~~~~~~~~~~~^ client=unprivileged_client, ^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...<9 lines>... external_route=True, ^^^^^^^^^^^^^^^^^^^^ ) as isvc: ^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'http-onnx-predictor-86778bd869-flmzk': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T13:59:41Z', 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T13:59:40Z', 'message': 'containers with incomplete status: [storage-initializer]', 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T13:59:40Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T13:59:40Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T13:59:40Z', 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'registry.redhat.io/rhoai/odh-openvino-model-server-rhel9@sha256:1ab58519c50e2c3a9ebf0fee6d0708b1b5a0ae972aefcc722d87b2f62239a033', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-jnt7j', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}, {'image': 'registry.redhat.io/rhoai/odh-kube-auth-proxy-rhel9@sha256:07ee0914946d9d6b2c7c7367be5b6bc13ddf0dddd93e896fc50aa617d761dc3a', 'imageID': '', 'lastState': {}, 'name': 'kube-rbac-proxy', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/etc/kube-rbac-proxy', 'name': 'http-onnx-kube-rbac-proxy-sar-config', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-jnt7j', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '192.169.1.109', 'hostIPs': [{'ip': '192.169.1.109'}], 'initContainerStatuses': [{'allocatedResources': {'cpu': '100m', 'memory': '100Mi'}, 'containerID': 'cri-o://cf54a7b53ff071f3c2078b2ae4782c7fcc1d59ec8a90452b58f56318810cade3', 'image': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:b73f3818b1c3b6f57d9cac6b975552c5888bf9c0d37df6c50938fd4988d2012b', 'imageID': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:9bb77298f77598b5a61a2aeaad625500a0e17b64e8da4b7a1988efb99bda2660', 'lastState': {'terminated': {'containerID': 'cri-o://1a034376d10b2df443a7296a115009cb3af084404bcc1b303d11f896134a6225', 'exitCode': 1, 'finishedAt': '2026-06-12T13:59:43Z', 'message': '2026-06-12 13:59:41.547 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' "(src_uri, dest_path): [('s3://ods-ci-s3/test-dir/', " "'/mnt/models')]\n" '2026-06-12 13:59:41.548 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://ods-ci-s3/test-dir/ to local\n' '2026-06-12 13:59:41.936 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():361] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-06-12 13:59:43.087 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():24] Unexpected error during ' 'storage initialization: SSL validation failed for ' 'https://s3.us-east-1.amazonaws.com/ods-ci-s3 [SSL: ' 'CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to ' 'get local issuer certificate (_ssl.c:1004)\n', 'reason': 'Error', 'startedAt': '2026-06-12T13:59:41Z'}}, 'name': 'storage-initializer', 'ready': False, 'resources': {'limits': {'cpu': '1', 'memory': '24Gi'}, 'requests': {'cpu': '100m', 'memory': '100Mi'}}, 'restartCount': 1, 'started': False, 'state': {'terminated': {'containerID': 'cri-o://cf54a7b53ff071f3c2078b2ae4782c7fcc1d59ec8a90452b58f56318810cade3', 'exitCode': 1, 'finishedAt': '2026-06-12T13:59:45Z', 'message': '2026-06-12 13:59:44.052 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' "(src_uri, dest_path): [('s3://ods-ci-s3/test-dir/', " "'/mnt/models')]\n" '2026-06-12 13:59:44.053 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://ods-ci-s3/test-dir/ to local\n' '2026-06-12 13:59:44.373 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():361] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-06-12 13:59:45.703 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():24] Unexpected error during ' 'storage initialization: SSL validation failed for ' 'https://s3.us-east-1.amazonaws.com/ods-ci-s3 [SSL: ' 'CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to ' 'get local issuer certificate (_ssl.c:1004)\n', 'reason': 'Error', 'startedAt': '2026-06-12T13:59:43Z'}}, 'user': {'linux': {'gid': 0, 'supplementalGroups': [0, 1001520000], 'uid': 1001520000}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-jnt7j', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'phase': 'Pending', 'podIP': '10.131.1.201', 'podIPs': [{'ip': '10.131.1.201'}], 'qosClass': 'Burstable', 'startTime': '2026-06-12T13:59:40Z'}}Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 353, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 245, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 165, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 523, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1723, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 707, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 539, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 627, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1091, in execute raise exc.with_traceback(exc_tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1202, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 908, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_server/kserve/authentication/conftest.py", line 144, in http_s3_ovms_raw_inference_service with create_isvc( ~~~~~~~~~~~^ client=unprivileged_client, ^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...<9 lines>... external_route=True, ^^^^^^^^^^^^^^^^^^^^ ) as isvc: ^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'http-onnx-predictor-86778bd869-flmzk': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T13:59:41Z', 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T13:59:40Z', 'message': 'containers with incomplete status: [storage-initializer]', 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T13:59:40Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T13:59:40Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T13:59:40Z', 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'registry.redhat.io/rhoai/odh-openvino-model-server-rhel9@sha256:1ab58519c50e2c3a9ebf0fee6d0708b1b5a0ae972aefcc722d87b2f62239a033', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-jnt7j', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}, {'image': 'registry.redhat.io/rhoai/odh-kube-auth-proxy-rhel9@sha256:07ee0914946d9d6b2c7c7367be5b6bc13ddf0dddd93e896fc50aa617d761dc3a', 'imageID': '', 'lastState': {}, 'name': 'kube-rbac-proxy', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/etc/kube-rbac-proxy', 'name': 'http-onnx-kube-rbac-proxy-sar-config', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-jnt7j', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '192.169.1.109', 'hostIPs': [{'ip': '192.169.1.109'}], 'initContainerStatuses': [{'allocatedResources': {'cpu': '100m', 'memory': '100Mi'}, 'containerID': 'cri-o://cf54a7b53ff071f3c2078b2ae4782c7fcc1d59ec8a90452b58f56318810cade3', 'image': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:b73f3818b1c3b6f57d9cac6b975552c5888bf9c0d37df6c50938fd4988d2012b', 'imageID': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:9bb77298f77598b5a61a2aeaad625500a0e17b64e8da4b7a1988efb99bda2660', 'lastState': {'terminated': {'containerID': 'cri-o://1a034376d10b2df443a7296a115009cb3af084404bcc1b303d11f896134a6225', 'exitCode': 1, 'finishedAt': '2026-06-12T13:59:43Z', 'message': '2026-06-12 13:59:41.547 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' "(src_uri, dest_path): [('s3://ods-ci-s3/test-dir/', " "'/mnt/models')]\n" '2026-06-12 13:59:41.548 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://ods-ci-s3/test-dir/ to local\n' '2026-06-12 13:59:41.936 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():361] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-06-12 13:59:43.087 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():24] Unexpected error during ' 'storage initialization: SSL validation failed for ' 'https://s3.us-east-1.amazonaws.com/ods-ci-s3 [SSL: ' 'CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to ' 'get local issuer certificate (_ssl.c:1004)\n', 'reason': 'Error', 'startedAt': '2026-06-12T13:59:41Z'}}, 'name': 'storage-initializer', 'ready': False, 'resources': {'limits': {'cpu': '1', 'memory': '24Gi'}, 'requests': {'cpu': '100m', 'memory': '100Mi'}}, 'restartCount': 1, 'started': False, 'state': {'terminated': {'containerID': 'cri-o://cf54a7b53ff071f3c2078b2ae4782c7fcc1d59ec8a90452b58f56318810cade3', 'exitCode': 1, 'finishedAt': '2026-06-12T13:59:45Z', 'message': '2026-06-12 13:59:44.052 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' "(src_uri, dest_path): [('s3://ods-ci-s3/test-dir/', " "'/mnt/models')]\n" '2026-06-12 13:59:44.053 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://ods-ci-s3/test-dir/ to local\n' '2026-06-12 13:59:44.373 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():361] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-06-12 13:59:45.703 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():24] Unexpected error during ' 'storage initialization: SSL validation failed for ' 'https://s3.us-east-1.amazonaws.com/ods-ci-s3 [SSL: ' 'CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to ' 'get local issuer certificate (_ssl.c:1004)\n', 'reason': 'Error', 'startedAt': '2026-06-12T13:59:43Z'}}, 'user': {'linux': {'gid': 0, 'supplementalGroups': [0, 1001520000], 'uid': 1001520000}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-jnt7j', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'phase': 'Pending', 'podIP': '10.131.1.201', 'podIPs': [{'ip': '10.131.1.201'}], 'qosClass': 'Burstable', 'startTime': '2026-06-12T13:59:40Z'}}Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 353, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 245, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 165, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 523, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1723, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 707, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 539, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 627, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1091, in execute raise exc.with_traceback(exc_tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1202, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 908, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_server/kserve/authentication/conftest.py", line 144, in http_s3_ovms_raw_inference_service with create_isvc( ~~~~~~~~~~~^ client=unprivileged_client, ^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...<9 lines>... external_route=True, ^^^^^^^^^^^^^^^^^^^^ ) as isvc: ^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'http-onnx-predictor-86778bd869-flmzk': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T13:59:41Z', 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T13:59:40Z', 'message': 'containers with incomplete status: [storage-initializer]', 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T13:59:40Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T13:59:40Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T13:59:40Z', 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'registry.redhat.io/rhoai/odh-openvino-model-server-rhel9@sha256:1ab58519c50e2c3a9ebf0fee6d0708b1b5a0ae972aefcc722d87b2f62239a033', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-jnt7j', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}, {'image': 'registry.redhat.io/rhoai/odh-kube-auth-proxy-rhel9@sha256:07ee0914946d9d6b2c7c7367be5b6bc13ddf0dddd93e896fc50aa617d761dc3a', 'imageID': '', 'lastState': {}, 'name': 'kube-rbac-proxy', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/etc/kube-rbac-proxy', 'name': 'http-onnx-kube-rbac-proxy-sar-config', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-jnt7j', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '192.169.1.109', 'hostIPs': [{'ip': '192.169.1.109'}], 'initContainerStatuses': [{'allocatedResources': {'cpu': '100m', 'memory': '100Mi'}, 'containerID': 'cri-o://cf54a7b53ff071f3c2078b2ae4782c7fcc1d59ec8a90452b58f56318810cade3', 'image': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:b73f3818b1c3b6f57d9cac6b975552c5888bf9c0d37df6c50938fd4988d2012b', 'imageID': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:9bb77298f77598b5a61a2aeaad625500a0e17b64e8da4b7a1988efb99bda2660', 'lastState': {'terminated': {'containerID': 'cri-o://1a034376d10b2df443a7296a115009cb3af084404bcc1b303d11f896134a6225', 'exitCode': 1, 'finishedAt': '2026-06-12T13:59:43Z', 'message': '2026-06-12 13:59:41.547 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' "(src_uri, dest_path): [('s3://ods-ci-s3/test-dir/', " "'/mnt/models')]\n" '2026-06-12 13:59:41.548 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://ods-ci-s3/test-dir/ to local\n' '2026-06-12 13:59:41.936 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():361] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-06-12 13:59:43.087 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():24] Unexpected error during ' 'storage initialization: SSL validation failed for ' 'https://s3.us-east-1.amazonaws.com/ods-ci-s3 [SSL: ' 'CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to ' 'get local issuer certificate (_ssl.c:1004)\n', 'reason': 'Error', 'startedAt': '2026-06-12T13:59:41Z'}}, 'name': 'storage-initializer', 'ready': False, 'resources': {'limits': {'cpu': '1', 'memory': '24Gi'}, 'requests': {'cpu': '100m', 'memory': '100Mi'}}, 'restartCount': 1, 'started': False, 'state': {'terminated': {'containerID': 'cri-o://cf54a7b53ff071f3c2078b2ae4782c7fcc1d59ec8a90452b58f56318810cade3', 'exitCode': 1, 'finishedAt': '2026-06-12T13:59:45Z', 'message': '2026-06-12 13:59:44.052 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' "(src_uri, dest_path): [('s3://ods-ci-s3/test-dir/', " "'/mnt/models')]\n" '2026-06-12 13:59:44.053 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://ods-ci-s3/test-dir/ to local\n' '2026-06-12 13:59:44.373 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():361] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-06-12 13:59:45.703 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():24] Unexpected error during ' 'storage initialization: SSL validation failed for ' 'https://s3.us-east-1.amazonaws.com/ods-ci-s3 [SSL: ' 'CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to ' 'get local issuer certificate (_ssl.c:1004)\n', 'reason': 'Error', 'startedAt': '2026-06-12T13:59:43Z'}}, 'user': {'linux': {'gid': 0, 'supplementalGroups': [0, 1001520000], 'uid': 1001520000}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-jnt7j', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'phase': 'Pending', 'podIP': '10.131.1.201', 'podIPs': [{'ip': '10.131.1.201'}], 'qosClass': 'Burstable', 'startTime': '2026-06-12T13:59:40Z'}}Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 353, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 245, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 165, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 523, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1723, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 707, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 539, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 627, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1110, in execute result: FixtureValue = ihook.pytest_fixture_setup( ~~~~~~~~~~~~~~~~~~~~~~~~~~^ fixturedef=self, request=request ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pytest_asyncio/plugin.py", line 728, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper return result.get_result() ~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result raise exc.with_traceback(tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper res = yield ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/setuponly.py", line 36, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1202, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 908, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_server/kserve/authentication/conftest.py", line 172, in http_s3_ovms_raw_inference_service_2 with create_isvc( ~~~~~~~~~~~^ client=unprivileged_client, ^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...<9 lines>... external_route=True, ^^^^^^^^^^^^^^^^^^^^ ) as isvc: ^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'http-onnx-2-predictor-84b9789455-c6r58': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T13:59:53Z', 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T13:59:52Z', 'message': 'containers with incomplete status: [storage-initializer]', 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T13:59:52Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T13:59:52Z', 'message': 'containers with unready status: [kserve-container ' 'kube-rbac-proxy]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T13:59:52Z', 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'registry.redhat.io/rhoai/odh-openvino-model-server-rhel9@sha256:1ab58519c50e2c3a9ebf0fee6d0708b1b5a0ae972aefcc722d87b2f62239a033', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-l7btj', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}, {'image': 'registry.redhat.io/rhoai/odh-kube-auth-proxy-rhel9@sha256:07ee0914946d9d6b2c7c7367be5b6bc13ddf0dddd93e896fc50aa617d761dc3a', 'imageID': '', 'lastState': {}, 'name': 'kube-rbac-proxy', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/etc/tls/private', 'name': 'proxy-tls'}, {'mountPath': '/etc/kube-rbac-proxy', 'name': 'http-onnx-2-kube-rbac-proxy-sar-config', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-l7btj', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '192.169.1.109', 'hostIPs': [{'ip': '192.169.1.109'}], 'initContainerStatuses': [{'allocatedResources': {'cpu': '100m', 'memory': '100Mi'}, 'containerID': 'cri-o://cbeee9bdc6bec3ed3ee630689adb6e673181fbee26f58c3ec039282d45b5b16b', 'image': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:b73f3818b1c3b6f57d9cac6b975552c5888bf9c0d37df6c50938fd4988d2012b', 'imageID': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:9bb77298f77598b5a61a2aeaad625500a0e17b64e8da4b7a1988efb99bda2660', 'lastState': {'terminated': {'containerID': 'cri-o://f560ba4757b8a9795937bc5cbcae6a8ce8b71a0ef8c6e08186574abbfc9e3fb8', 'exitCode': 1, 'finishedAt': '2026-06-12T13:59:59Z', 'message': '2026-06-12 13:59:53.541 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' "(src_uri, dest_path): [('s3://ods-ci-s3/test-dir/', " "'/mnt/models')]\n" '2026-06-12 13:59:53.543 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://ods-ci-s3/test-dir/ to local\n' '2026-06-12 13:59:53.859 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():361] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-06-12 13:59:58.934 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():24] Unexpected error during ' 'storage initialization: SSL validation failed for ' 'https://s3.us-east-1.amazonaws.com/ods-ci-s3 [SSL: ' 'CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to ' 'get local issuer certificate (_ssl.c:1004)\n', 'reason': 'Error', 'startedAt': '2026-06-12T13:59:53Z'}}, 'name': 'storage-initializer', 'ready': False, 'resources': {'limits': {'cpu': '1', 'memory': '24Gi'}, 'requests': {'cpu': '100m', 'memory': '100Mi'}}, 'restartCount': 1, 'started': False, 'state': {'terminated': {'containerID': 'cri-o://cbeee9bdc6bec3ed3ee630689adb6e673181fbee26f58c3ec039282d45b5b16b', 'exitCode': 1, 'finishedAt': '2026-06-12T14:00:04Z', 'message': '2026-06-12 14:00:00.139 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' "(src_uri, dest_path): [('s3://ods-ci-s3/test-dir/', " "'/mnt/models')]\n" '2026-06-12 14:00:00.139 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://ods-ci-s3/test-dir/ to local\n' '2026-06-12 14:00:00.447 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():361] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-06-12 14:00:04.242 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():24] Unexpected error during ' 'storage initialization: SSL validation failed for ' 'https://s3.us-east-1.amazonaws.com/ods-ci-s3 [SSL: ' 'CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to ' 'get local issuer certificate (_ssl.c:1004)\n', 'reason': 'Error', 'startedAt': '2026-06-12T13:59:59Z'}}, 'user': {'linux': {'gid': 0, 'supplementalGroups': [0, 1001520000], 'uid': 1001520000}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}, {'mountPath': '/var/run/secrets/kubernetes.io/serviceaccount', 'name': 'kube-api-access-l7btj', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'phase': 'Pending', 'podIP': '10.131.1.202', 'podIPs': [{'ip': '10.131.1.202'}], 'qosClass': 'Burstable', 'startTime': '2026-06-12T13:59:52Z'}}Traceback (most recent call last): File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 353, in from_call result: TResult | None = func() ~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 245, in <lambda> lambda: runtest_hook(item=item, **kwds), ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 165, in pytest_runtest_setup item.session._setupstate.setup(item) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 523, in setup col.setup() ~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1723, in setup self._request._fillfixtures() ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 707, in _fillfixtures item.funcargs[argname] = self.getfixturevalue(argname) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 539, in getfixturevalue fixturedef = self._get_active_fixturedef(argname) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 627, in _get_active_fixturedef fixturedef.execute(request=subrequest) ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1110, in execute result: FixtureValue = ihook.pytest_fixture_setup( ~~~~~~~~~~~~~~~~~~~~~~~~~~^ fixturedef=self, request=request ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall raise exception File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pytest_asyncio/plugin.py", line 728, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper return result.get_result() ~~~~~~~~~~~~~~~~~^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result raise exc.with_traceback(tb) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper res = yield ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall teardown.throw(exception) ~~~~~~~~~~~~~~^^^^^^^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/setuponly.py", line 36, in pytest_fixture_setup return (yield) ^^^^^ File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall res = hook_impl.function(*args) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1202, in pytest_fixture_setup result = call_fixture_func(fixturefunc, request, kwargs) File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 908, in call_fixture_func fixture_result = next(generator) File "/home/odh/opendatahub-tests/tests/model_serving/model_server/kserve/authentication/conftest.py", line 214, in unprivileged_s3_ovms_raw_inference_service with create_isvc( ~~~~~~~~~~~^ client=unprivileged_client, ^^^^^^^^^^^^^^^^^^^^^^^^^^^ ...<6 lines>... storage_path=request.param["model-dir"], ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) as isvc: ^ File "/usr/lib64/python3.14/contextlib.py", line 141, in __enter__ return next(self.gen) File "/home/odh/opendatahub-tests/utilities/inference_utils.py", line 760, in create_isvc verify_no_failed_pods( ~~~~~~~~~~~~~~~~~~~~~^ client=client, ^^^^^^^^^^^^^^ ...<2 lines>... timeout=timeout_watch.remaining_time(), ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ) ^ File "/home/odh/opendatahub-tests/utilities/infra.py", line 804, in verify_no_failed_pods raise FailedPodsError(pods=failed_pods) utilities.exceptions.FailedPodsError: The following pods are not running: {'http-onnx-raw-predictor-7d584db466-2dq7k': {'conditions': [{'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:00:31Z', 'status': 'True', 'type': 'PodReadyToStartContainers'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:00:30Z', 'message': 'containers with incomplete status: [storage-initializer]', 'reason': 'ContainersNotInitialized', 'status': 'False', 'type': 'Initialized'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:00:30Z', 'message': 'containers with unready status: [kserve-container]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'Ready'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:00:30Z', 'message': 'containers with unready status: [kserve-container]', 'reason': 'ContainersNotReady', 'status': 'False', 'type': 'ContainersReady'}, {'lastProbeTime': None, 'lastTransitionTime': '2026-06-12T14:00:30Z', 'status': 'True', 'type': 'PodScheduled'}], 'containerStatuses': [{'image': 'registry.redhat.io/rhoai/odh-openvino-model-server-rhel9@sha256:1ab58519c50e2c3a9ebf0fee6d0708b1b5a0ae972aefcc722d87b2f62239a033', 'imageID': '', 'lastState': {}, 'name': 'kserve-container', 'ready': False, 'restartCount': 0, 'started': False, 'state': {'waiting': {'reason': 'PodInitializing'}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'hostIP': '192.169.1.109', 'hostIPs': [{'ip': '192.169.1.109'}], 'initContainerStatuses': [{'allocatedResources': {'cpu': '100m', 'memory': '100Mi'}, 'containerID': 'cri-o://ee342661c2e61b3578e6aeaed31b5f06cd798c61d0c55c04379bcf755fad59dd', 'image': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:b73f3818b1c3b6f57d9cac6b975552c5888bf9c0d37df6c50938fd4988d2012b', 'imageID': 'registry.redhat.io/rhoai/odh-kserve-storage-initializer-rhel9@sha256:9bb77298f77598b5a61a2aeaad625500a0e17b64e8da4b7a1988efb99bda2660', 'lastState': {'terminated': {'containerID': 'cri-o://41cdb8a3ca6bc8724eb28d1a607ad0da42d1403a0fe5f8b62b704fedcfab3b73', 'exitCode': 1, 'finishedAt': '2026-06-12T14:00:35Z', 'message': '2026-06-12 14:00:31.743 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' "(src_uri, dest_path): [('s3://ods-ci-s3/test-dir/', " "'/mnt/models')]\n" '2026-06-12 14:00:31.744 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://ods-ci-s3/test-dir/ to local\n' '2026-06-12 14:00:32.035 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():361] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-06-12 14:00:35.550 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():24] Unexpected error during ' 'storage initialization: SSL validation failed for ' 'https://s3.us-east-1.amazonaws.com/ods-ci-s3 [SSL: ' 'CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to ' 'get local issuer certificate (_ssl.c:1004)\n', 'reason': 'Error', 'startedAt': '2026-06-12T14:00:31Z'}}, 'name': 'storage-initializer', 'ready': False, 'resources': {'limits': {'cpu': '1', 'memory': '24Gi'}, 'requests': {'cpu': '100m', 'memory': '100Mi'}}, 'restartCount': 1, 'started': False, 'state': {'terminated': {'containerID': 'cri-o://ee342661c2e61b3578e6aeaed31b5f06cd798c61d0c55c04379bcf755fad59dd', 'exitCode': 1, 'finishedAt': '2026-06-12T14:00:40Z', 'message': '2026-06-12 14:00:36.441 1 storage.initializer INFO ' '[initializer-entrypoint:<module>():16] Initializing, args: ' "(src_uri, dest_path): [('s3://ods-ci-s3/test-dir/', " "'/mnt/models')]\n" '2026-06-12 14:00:36.441 1 storage.initializer INFO ' '[kserve_storage.py:download():161] Copying contents of ' 's3://ods-ci-s3/test-dir/ to local\n' '2026-06-12 14:00:36.730 1 storage.initializer INFO ' '[kserve_storage.py:_get_s3_client_kwargs():361] ca bundle ' 'file(/etc/ssl/custom-certs/cabundle.crt) exists.\n' '2026-06-12 14:00:40.059 1 storage.initializer ERROR ' '[initializer-entrypoint:<module>():24] Unexpected error during ' 'storage initialization: SSL validation failed for ' 'https://s3.us-east-1.amazonaws.com/ods-ci-s3 [SSL: ' 'CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to ' 'get local issuer certificate (_ssl.c:1004)\n', 'reason': 'Error', 'startedAt': '2026-06-12T14:00:36Z'}}, 'user': {'linux': {'gid': 0, 'supplementalGroups': [0, 1001540000], 'uid': 1001540000}}, 'volumeMounts': [{'mountPath': '/mnt/models', 'name': 'kserve-provision-location'}, {'mountPath': '/etc/ssl/custom-certs', 'name': 'cabundle-cert', 'readOnly': True, 'recursiveReadOnly': 'Disabled'}]}], 'phase': 'Pending', 'podIP': '10.131.1.204', 'podIPs': [{'ip': '10.131.1.204'}], 'qosClass': 'Burstable', 'startTime': '2026-06-12T14:00:30Z'}}