--- BVT pytest attempt 2026-06-14T09:32:42.758664+00:00 (/.local/bin/uv run pytest...) --- 2026-06-14T09:32:44.371631Z [info ] Writing tests log to /home/odh/opendatahub-tests/results/pytest-tests.log [conftest] 2026-06-14T09:32:44.381967Z [info ] BEGIN (implicit)  [sqlalchemy.engine.Engine] 2026-06-14T09:32:44.382232Z [info ] BEGIN (implicit)  [sqlalchemy.engine.Engine] 2026-06-14T09:32:44.382371Z [info ] PRAGMA main.table_info("OpenDataHubTestTable") [sqlalchemy.engine.Engine] 2026-06-14T09:32:44.382511Z [info ] PRAGMA main.table_info("OpenDataHubTestTable") [sqlalchemy.engine.Engine] 2026-06-14T09:32:44.382666Z [info ] [raw sql] ()  [sqlalchemy.engine.Engine] 2026-06-14T09:32:44.382811Z [info ] [raw sql] ()  [sqlalchemy.engine.Engine] 2026-06-14T09:32:44.383927Z [info ] PRAGMA temp.table_info("OpenDataHubTestTable") [sqlalchemy.engine.Engine] 2026-06-14T09:32:44.384455Z [info ] PRAGMA temp.table_info("OpenDataHubTestTable") [sqlalchemy.engine.Engine] 2026-06-14T09:32:44.384607Z [info ] [raw sql] ()  [sqlalchemy.engine.Engine] 2026-06-14T09:32:44.384737Z [info ] [raw sql] ()  [sqlalchemy.engine.Engine] 2026-06-14T09:32:44.384867Z [info ]  CREATE TABLE "OpenDataHubTestTable" ( id INTEGER NOT NULL, test_name VARCHAR(500) NOT NULL, start_time INTEGER NOT NULL, PRIMARY KEY (id) )  [sqlalchemy.engine.Engine] 2026-06-14T09:32:44.384987Z [info ]  CREATE TABLE "OpenDataHubTestTable" ( id INTEGER NOT NULL, test_name VARCHAR(500) NOT NULL, start_time INTEGER NOT NULL, PRIMARY KEY (id) )  [sqlalchemy.engine.Engine] 2026-06-14T09:32:44.385143Z [info ] [no key 0.00005s] ()  [sqlalchemy.engine.Engine] 2026-06-14T09:32:44.385274Z [info ] [no key 0.00005s] ()  [sqlalchemy.engine.Engine] 2026-06-14T09:32:44.385838Z [info ] COMMIT  [sqlalchemy.engine.Engine] 2026-06-14T09:32:44.385968Z [info ] COMMIT  [sqlalchemy.engine.Engine] 2026-06-14T09:32:44.388774Z [info ] Trying to get client via new_client_from_config [ocp_resources.resource] 2026-06-14T09:32:44.393687Z [info ] kind: DataScienceCluster api version: datasciencecluster.opendatahub.io/v2 [ocp_resources.resource] /home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/urllib3/connectionpool.py:1110: InsecureRequestWarning: Unverified HTTPS request is being made to host 'api.ods-qe-psi-23.osp.rh-ods.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings warnings.warn( /home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/urllib3/connectionpool.py:1110: InsecureRequestWarning: Unverified HTTPS request is being made to host 'api.ods-qe-psi-23.osp.rh-ods.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings warnings.warn( /home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/urllib3/connectionpool.py:1110: InsecureRequestWarning: Unverified HTTPS request is being made to host 'api.ods-qe-psi-23.osp.rh-ods.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings warnings.warn( 2026-06-14T09:32:44.476187Z [info ] kind: DSCInitialization api version: dscinitialization.opendatahub.io/v2 [ocp_resources.resource] /home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/urllib3/connectionpool.py:1110: InsecureRequestWarning: Unverified HTTPS request is being made to host 'api.ods-qe-psi-23.osp.rh-ods.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings warnings.warn( ============================= test session starts ============================== platform linux -- Python 3.14.5, pytest-9.0.3, pluggy-1.6.0 -- /home/odh/opendatahub-tests/.venv/bin/python cachedir: .pytest_cache metadata: {'Python': '3.14.5', 'Platform': 'Linux-5.14.0-427.115.1.el9_4.x86_64-x86_64-with-glibc2.42', 'Packages': {'pytest': '9.0.3', 'pluggy': '1.6.0'}, 'Plugins': {'anyio': '4.13.0', 'langsmith': '0.8.9', 'asyncio': '1.4.0', 'dependency': '0.6.1', 'html': '4.2.0', 'metadata': '3.1.1', 'order': '1.4.0', 'progress': '1.4.0', 'testconfig': '0.2.0', 'xdist': '3.8.0', 'syrupy': '5.3.1'}} rootdir: /home/odh/opendatahub-tests configfile: pytest.ini plugins: anyio-4.13.0, langsmith-0.8.9, asyncio-1.4.0, dependency-0.6.1, html-4.2.0, metadata-3.1.1, order-1.4.0, progress-1.4.0, testconfig-0.2.0, xdist-3.8.0, syrupy-5.3.1 asyncio: mode=Mode.STRICT, debug=False, asyncio_default_fixture_loop_scope=None, asyncio_default_test_loop_scope=function collecting ... collected 35 items / 28 deselected / 7 selected tests/ogx/operator/test_ogx_server.py::TestOgxServer::test_ogx_server_verify_images[unprivileged_model_namespace0-ogx_server0]  ----------------------- test_ogx_server_verify_images[unprivileged_model_namespace0-ogx_server0] ----------------------- -------------------------------------------------------- SETUP -------------------------------------------------------- 2026-06-14T09:32:45.935038Z [info ] (OpenDataHubTestTable|OpenDataHubTestTable) _post_configure_properties() started [sqlalchemy.orm.mapper.Mapper] 2026-06-14T09:32:45.935280Z [info ] (OpenDataHubTestTable|OpenDataHubTestTable) initialize prop id [sqlalchemy.orm.mapper.Mapper] 2026-06-14T09:32:45.935415Z [info ] (OpenDataHubTestTable|OpenDataHubTestTable) initialize prop test_name [sqlalchemy.orm.mapper.Mapper] 2026-06-14T09:32:45.936043Z [info ] (OpenDataHubTestTable|OpenDataHubTestTable) initialize prop start_time [sqlalchemy.orm.mapper.Mapper] 2026-06-14T09:32:45.936677Z [info ] (OpenDataHubTestTable|OpenDataHubTestTable) _post_configure_properties() complete [sqlalchemy.orm.mapper.Mapper] 2026-06-14T09:32:45.937178Z [info ] BEGIN (implicit)  [sqlalchemy.engine.Engine] 2026-06-14T09:32:45.937306Z [info ] BEGIN (implicit)  [sqlalchemy.engine.Engine] 2026-06-14T09:32:45.937430Z [info ] INSERT INTO "OpenDataHubTestTable" (test_name, start_time) VALUES (?, ?) [sqlalchemy.engine.Engine] 2026-06-14T09:32:45.937565Z [info ] INSERT INTO "OpenDataHubTestTable" (test_name, start_time) VALUES (?, ?) [sqlalchemy.engine.Engine] 2026-06-14T09:32:45.937718Z [info ] [generated in 0.00013s] ('/home/odh/opendatahub-tests/tests/ogx/operator/test_ogx_server.py::test_ogx_server_verify_images[unprivileged_model_namespace0-ogx_server0]', 1781429565) [sqlalchemy.engine.Engine] 2026-06-14T09:32:45.937866Z [info ] [generated in 0.00013s] ('/home/odh/opendatahub-tests/tests/ogx/operator/test_ogx_server.py::test_ogx_server_verify_images[unprivileged_model_namespace0-ogx_server0]', 1781429565) [sqlalchemy.engine.Engine] 2026-06-14T09:32:45.937994Z [info ] COMMIT  [sqlalchemy.engine.Engine] 2026-06-14T09:32:45.939023Z [info ] COMMIT  [sqlalchemy.engine.Engine] 2026-06-14T09:32:45.941202Z [info ] Executing session fixture: event_loop_policy [conftest] 2026-06-14T09:32:45.941575Z [info ] Executing session fixture: _syrupy_apply_ide_patches [conftest] 2026-06-14T09:32:45.945923Z [info ] Executing session fixture: admin_client [conftest] 2026-06-14T09:32:45.946213Z [info ] Trying to get client via new_client_from_config [ocp_resources.resource] 2026-06-14T09:32:45.946332Z [info ] Executing session fixture: dsc_resource [conftest] 2026-06-14T09:32:45.946465Z [info ] kind: DataScienceCluster api version: datasciencecluster.opendatahub.io/v2 [ocp_resources.resource] 2026-06-14T09:32:46.017067Z [info ] Executing session fixture: tmp_path_factory [conftest] 2026-06-14T09:32:46.017359Z [info ] Executing session fixture: tests_tmp_dir [conftest] 2026-06-14T09:32:46.018151Z [info ] Executing session fixture: os_path_environment [conftest] 2026-06-14T09:32:46.018275Z [info ] Executing session fixture: tmpdir_factory [conftest] 2026-06-14T09:32:46.018388Z [info ] Executing session fixture: bin_directory [conftest] 2026-06-14T09:32:46.019645Z [info ] Executing session fixture: oc_binary_path [conftest] 2026-06-14T09:32:46.019788Z [info ] kind: ConsoleCLIDownload api version: console.openshift.io/v1 [ocp_resources.resource] 2026-06-14T09:32:46.041235Z [info ] All oc console cli download links: ['https://downloads-openshift-console.apps.ods-qe-psi-23.osp.rh-ods.com/amd64/linux/oc.tar'] [utilities.infra] name=utilities.infra 2026-06-14T09:32:46.041309Z [info ] Downloading archive using: url=https://downloads-openshift-console.apps.ods-qe-psi-23.osp.rh-ods.com/amd64/linux/oc.tar [utilities.infra] name=utilities.infra 2026-06-14T09:32:54.218516Z [info ] Extract the downloaded archive. [utilities.infra] name=utilities.infra 2026-06-14T09:32:54.530947Z [info ] Downloaded file: ['oc']  [utilities.infra] name=utilities.infra 2026-06-14T09:32:54.566533Z [info ] Executing session fixture: bin_directory_to_os_path [conftest] 2026-06-14T09:32:54.562152Z [info ] OC binary path: /tmp/pytest-kLmr6ud42aNwHXTzQGu9Eq/bin0/oc [tests.conftest] name=tests.conftest 2026-06-14T09:32:54.562251Z [info ] Adding /tmp/pytest-kLmr6ud42aNwHXTzQGu9Eq/bin0 to $PATH [tests.conftest] name=tests.conftest 2026-06-14T09:32:54.567179Z [info ] Executing session fixture: nodes [conftest] 2026-06-14T09:32:54.609480Z [info ] Executing session fixture: dsci_resource [conftest] 2026-06-14T09:32:54.609636Z [info ] kind: DSCInitialization api version: dscinitialization.opendatahub.io/v2 [ocp_resources.resource] 2026-06-14T09:32:54.623151Z [info ] Executing session fixture: record_testsuite_property [conftest] 2026-06-14T09:32:54.623301Z [info ] Executing session fixture: junitxml_plugin [conftest] 2026-06-14T09:32:54.623439Z [info ] Executing session fixture: cluster_sanity_scope_session [conftest] 2026-06-14T09:32:54.622473Z [info ] Check cluster sanity.  [utilities.infra] name=utilities.infra 2026-06-14T09:32:54.623662Z [info ] Verify all nodes are in a healthy condition. [ocp_utilities.infra] 2026-06-14T09:32:54.693358Z [info ] Verify all nodes are schedulable. [ocp_utilities.infra] 2026-06-14T09:32:54.762450Z [info ] Waiting for 120 seconds [0:02:00], retry every 5 seconds. (Function: utilities.infra.wait_for_dsci_status_ready Kwargs: {'dsci_resource': }) [timeout_sampler] 2026-06-14T09:32:54.761889Z [info ] Wait for DSCI default-dsci to be in Ready status. [utilities.infra] name=utilities.infra 2026-06-14T09:32:54.762752Z [info ] Get DSCInitialization default-dsci status [ocp_resources DSCInitialization] 2026-06-14T09:32:54.776216Z [info ] Elapsed time: 0.0001614093780517578 [0:00:00.000161] [timeout_sampler] 2026-06-14T09:32:54.776412Z [info ] Waiting for 120 seconds [0:02:00], retry every 5 seconds. (Function: utilities.infra.wait_for_dsc_status_ready Kwargs: {'dsc_resource': }) [timeout_sampler] 2026-06-14T09:32:54.775733Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:32:54.776637Z [info ] Get DataScienceCluster default-dsc status [ocp_resources DataScienceCluster] 2026-06-14T09:32:59.809150Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:33:04.848177Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:33:09.888239Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:33:14.928279Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:33:19.968517Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:33:25.007581Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:33:30.047976Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:33:35.087223Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:33:40.127337Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:33:45.173465Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:33:50.213220Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:33:55.255621Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:34:00.296700Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:34:05.335907Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:34:10.380203Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:34:15.419314Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:34:20.458290Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:34:25.499732Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:34:30.538942Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:34:35.577190Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:34:40.616719Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:34:45.657168Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:34:50.696971Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:34:55.737729Z [error ] Cluster sanity check failed: Timed Out: 120 Function: utilities.infra.wait_for_dsc_status_ready Kwargs: {'dsc_resource': } Last exception: ResourceNotReadyError: DSC default-dsc is not ready. Current status: {'components': {'aipipelines': {'managementState': 'Removed'}, 'dashboard': {'managementState': 'Managed'}, 'feastoperator': {'managementState': 'Removed'}, 'kserve': {'managementState': 'Managed', 'releases': [{'name': 'KServe', 'repoUrl': 'https://github.com/kserve/kserve/', 'version': 'v0.17.0'}, {'name': 'vLLM', 'repoUrl': 'https://github.com/vllm-project/vllm', 'version': 'v0.18.0'}, {'name': 'llm-d-inference-scheduler', 'repoUrl': 'https://github.com/llm-d/llm-d-inference-scheduler', 'version': 'v0.7.1'}, {'name': 'llm-d-workload-variant-autoscaler', 'repoUrl': 'https://github.com/llm-d/llm-d-workload-variant-autoscaler', 'version': 'v0.6.0'}]}, 'kueue': {'managementState': 'Removed'}, 'llamastackoperator': {'managementState': 'Removed'}, 'mlflowoperator': {'managementState': 'Removed'}, 'modelregistry': {'managementState': 'Managed', 'registriesNamespace': 'rhoai-model-registries', 'releases': [{'name': 'Kubeflow Model Registry', 'repoUrl': 'https://github.com/kubeflow/model-registry', 'version': 'latest'}, {'name': 'Open Data Hub Model Registry Operator', 'repoUrl': 'https://github.com/opendatahub-io/model-registry-operator', 'version': 'latest'}]}, 'ray': {'managementState': 'Managed', 'releases': [{'name': 'KubeRay', 'repoUrl': 'https://github.com/ray-project/kuberay', 'version': '1.4.2'}]}, 'sparkoperator': {'managementState': 'Removed'}, 'trainer': {'managementState': 'Removed'}, 'trainingoperator': {'managementState': 'Managed', 'releases': [{'name': 'Kubeflow Training Operator', 'repoUrl': 'https://github.com/kubeflow/trainer', 'version': '1.9.0'}]}, 'trustyai': {'managementState': 'Managed', 'releases': [{'name': 'TrustyAI operator', 'repoUrl': 'https://github.com/trustyai-explainability/trustyai-service-operator', 'version': 'v1.37.0'}, {'name': 'TrustyAI service', 'repoUrl': 'https://github.com/trustyai-explainability/trustyai-explainability', 'version': 'v0.28.0'}, {'name': 'TrustyAI LMEval driver', 'repoUrl': 'https://github.com/trustyai-explainability/trustyai-service-operator', 'version': 'v1.37.0'}, {'name': 'TrustyAI LMEval job', 'repoUrl': 'https://github.com/EleutherAI/lm-evaluation-harness', 'version': 'v0.4.8'}, {'name': 'TrustyAI Guardrails orchestrator', 'repoUrl': 'https://github.com/foundation-model-stack/fms-guardrails-orchestrator', 'version': '0.9.4'}, {'name': 'TrustyAI builtin detectors', 'repoUrl': 'https://github.com/trustyai-explainability/guardrails-regex-detector', 'version': 'v0.2.0'}, {'name': 'TrustyAI sidecar gateway', 'repoUrl': 'https://github.com/trustyai-explainability/vllm-orchestrator-gateway', 'version': 'v0.2.1'}]}, 'workbenches': {'managementState': 'Managed', 'releases': [{'name': 'Kubeflow Notebook Controller', 'repoUrl': 'https://github.com/kubeflow/kubeflow', 'version': '1.10.0'}], 'workbenchNamespace': 'rhods-notebooks'}}, 'conditions': [{'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'failure deploying resource /default-kserve: apply failed ' 'components.platform.opendatahub.io/v1alpha1, Kind=Kserve: unable ' 'to patch components.platform.opendatahub.io/v1alpha1, Kind=Kserve ' 'default-kserve: Kserve.components.platform.opendatahub.io ' '"default-kserve" is invalid: [spec.modelsAsService: Invalid ' 'value: "null": spec.modelsAsService in body must be of type ' 'object: "null", : Invalid value: "null": some validation ' 'rules were not checked because the object was invalid; correct ' 'the existing errors to complete validation]', 'reason': 'Error', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'failure deploying resource /default-kserve: apply failed ' 'components.platform.opendatahub.io/v1alpha1, Kind=Kserve: unable ' 'to patch components.platform.opendatahub.io/v1alpha1, Kind=Kserve ' 'default-kserve: Kserve.components.platform.opendatahub.io ' '"default-kserve" is invalid: [spec.modelsAsService: Invalid ' 'value: "null": spec.modelsAsService in body must be of type ' 'object: "null", : Invalid value: "null": some validation ' 'rules were not checked because the object was invalid; correct ' 'the existing errors to complete validation]', 'observedGeneration': 3, 'reason': 'Error', 'status': 'False', 'type': 'ProvisioningSucceeded'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'ComponentsReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'AIPipelinesReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'DashboardReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'FeastOperatorReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'KserveLLMInferenceServiceDependencies'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'KserveLLMInferenceServiceWideEPDependencies'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'KserveReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'KueueReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'LlamaStackOperatorReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'MLflowOperatorReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'ModelControllerReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'ModelRegistryReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'ModelsAsServiceReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'RayReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'SparkOperatorReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'TrainerReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'TrainingOperatorReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'TrustyAIReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'WorkbenchesReady'}], 'observedGeneration': 1, 'phase': 'Not Ready', 'release': {'name': 'OpenShift AI Self-Managed', 'version': '3.4.0'}}. [utilities.infra] name=utilities.infra =============== 28 deselected, 11 warnings in 131.87s (0:02:11) ================ =============== 28 deselected, 11 warnings in 131.87s (0:02:11) ================ ! _pytest.outcomes.Exit: Cluster sanity check failed: Timed Out: 120 Function: utilities.infra.wait_for_dsc_status_ready Kwargs: {'dsc_resource': } Last exception: ResourceNotReadyError: DSC default-dsc is not ready. Current status: {'components': {'aipipelines': {'managementState': 'Removed'}, 'dashboard': {'managementState': 'Managed'}, 'feastoperator': {'managementState': 'Removed'}, 'kserve': {'managementState': 'Managed', 'releases': [{'name': 'KServe', 'repoUrl': 'https://github.com/kserve/kserve/', 'version': 'v0.17.0'}, {'name': 'vLLM', 'repoUrl': 'https://github.com/vllm-project/vllm', 'version': 'v0.18.0'}, {'name': 'llm-d-inference-scheduler', 'repoUrl': 'https://github.com/llm-d/llm-d-inference-scheduler', 'version': 'v0.7.1'}, {'name': 'llm-d-workload-variant-autoscaler', 'repoUrl': 'https://github.com/llm-d/llm-d-workload-variant-autoscaler', 'version': 'v0.6.0'}]}, 'kueue': {'managementState': 'Removed'}, 'llamastackoperator': {'managementState': 'Removed'}, 'mlflowoperator': {'managementState': 'Removed'}, 'modelregistry': {'managementState': 'Managed', 'registriesNamespace': 'rhoai-model-registries', 'releases': [{'name': 'Kubeflow Model Registry', 'repoUrl': 'https://github.com/kubeflow/model-registry', 'version': 'latest'}, {'name': 'Open Data Hub Model Registry Operator', 'repoUrl': 'https://github.com/opendatahub-io/model-registry-operator', 'version': 'latest'}]}, 'ray': {'managementState': 'Managed', 'releases': [{'name': 'KubeRay', 'repoUrl': 'https://github.com/ray-project/kuberay', 'version': '1.4.2'}]}, 'sparkoperator': {'managementState': 'Removed'}, 'trainer': {'managementState': 'Removed'}, 'trainingoperator': {'managementState': 'Managed', 'releases': [{'name': 'Kubeflow Training Operator', 'repoUrl': 'https://github.com/kubeflow/trainer', 'version': '1.9.0'}]}, 'trustyai': {'managementState': 'Managed', 'releases': [{'name': 'TrustyAI operator', 'repoUrl': 'https://github.com/trustyai-explainability/trustyai-service-operator', 'version': 'v1.37.0'}, {'name': 'TrustyAI service', 'repoUrl': 'https://github.com/trustyai-explainability/trustyai-explainability', 'version': 'v0.28.0'}, {'name': 'TrustyAI LMEval driver', 'repoUrl': 'https://github.com/trustyai-explainability/trustyai-service-operator', 'version': 'v1.37.0'}, {'name': 'TrustyAI LMEval job', 'repoUrl': 'https://github.com/EleutherAI/lm-evaluation-harness', 'version': 'v0.4.8'}, {'name': 'TrustyAI Guardrails orchestrator', 'repoUrl': 'https://github.com/foundation-model-stack/fms-guardrails-orchestrator', 'version': '0.9.4'}, {'name': 'TrustyAI builtin detectors', 'repoUrl': 'https://github.com/trustyai-explainability/guardrails-regex-detector', 'version': 'v0.2.0'}, {'name': 'TrustyAI sidecar gateway', 'repoUrl': 'https://github.com/trustyai-explainability/vllm-orchestrator-gateway', 'version': 'v0.2.1'}]}, 'workbenches': {'managementState': 'Managed', 'releases': [{'name': 'Kubeflow Notebook Controller', 'repoUrl': 'https://github.com/kubeflow/kubeflow', 'version': '1.10.0'}], 'workbenchNamespace': 'rhods-notebooks'}}, 'conditions': [{'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'failure deploying resource /default-kserve: apply failed ' 'components.platform.opendatahub.io/v1alpha1, Kind=Kserve: unable ' 'to patch components.platform.opendatahub.io/v1alpha1, Kind=Kserve ' 'default-kserve: Kserve.components.platform.opendatahub.io ' '"default-kserve" is invalid: [spec.modelsAsService: Invalid ' 'value: "null": spec.modelsAsService in body must be of type ' 'object: "null", : Invalid value: "null": some validation ' 'rules were not checked because the object was invalid; correct ' 'the existing errors to complete validation]', 'reason': 'Error', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'failure deploying resource /default-kserve: apply failed ' 'components.platform.opendatahub.io/v1alpha1, Kind=Kserve: unable ' 'to patch components.platform.opendatahub.io/v1alpha1, Kind=Kserve ' 'default-kserve: Kserve.components.platform.opendatahub.io ' '"default-kserve" is invalid: [spec.modelsAsService: Invalid ' 'value: "null": spec.modelsAsService in body must be of type ' 'object: "null", : Invalid value: "null": some validation ' 'rules were not checked because the object was invalid; correct ' 'the existing errors to complete validation]', 'observedGeneration': 3, 'reason': 'Error', 'status': 'False', 'type': 'ProvisioningSucceeded'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'ComponentsReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'AIPipelinesReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'DashboardReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'FeastOperatorReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'KserveLLMInferenceServiceDependencies'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'KserveLLMInferenceServiceWideEPDependencies'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'KserveReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'KueueReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'LlamaStackOperatorReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'MLflowOperatorReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'ModelControllerReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'ModelRegistryReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'ModelsAsServiceReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'RayReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'SparkOperatorReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'TrainerReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'TrainingOperatorReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'TrustyAIReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'WorkbenchesReady'}], 'observedGeneration': 1, 'phase': 'Not Ready', 'release': {'name': 'OpenShift AI Self-Managed', 'version': '3.4.0'}}. ! --- BVT pytest attempt 2026-06-14T09:34:57.926418+00:00 (/.local/bin/uv run pytest...) --- Bytecode compiled 16343 files in 578ms 2026-06-14T09:35:00.129300Z [info ] Writing tests log to /home/odh/opendatahub-tests/results/pytest-tests.log [conftest] 2026-06-14T09:35:00.138453Z [info ] BEGIN (implicit)  [sqlalchemy.engine.Engine] 2026-06-14T09:35:00.138709Z [info ] BEGIN (implicit)  [sqlalchemy.engine.Engine] 2026-06-14T09:35:00.139179Z [info ] PRAGMA main.table_info("OpenDataHubTestTable") [sqlalchemy.engine.Engine] 2026-06-14T09:35:00.139881Z [info ] PRAGMA main.table_info("OpenDataHubTestTable") [sqlalchemy.engine.Engine] 2026-06-14T09:35:00.140126Z [info ] [raw sql] ()  [sqlalchemy.engine.Engine] 2026-06-14T09:35:00.140294Z [info ] [raw sql] ()  [sqlalchemy.engine.Engine] 2026-06-14T09:35:00.140450Z [info ] PRAGMA temp.table_info("OpenDataHubTestTable") [sqlalchemy.engine.Engine] 2026-06-14T09:35:00.140758Z [info ] PRAGMA temp.table_info("OpenDataHubTestTable") [sqlalchemy.engine.Engine] 2026-06-14T09:35:00.140928Z [info ] [raw sql] ()  [sqlalchemy.engine.Engine] 2026-06-14T09:35:00.141130Z [info ] [raw sql] ()  [sqlalchemy.engine.Engine] 2026-06-14T09:35:00.141293Z [info ]  CREATE TABLE "OpenDataHubTestTable" ( id INTEGER NOT NULL, test_name VARCHAR(500) NOT NULL, start_time INTEGER NOT NULL, PRIMARY KEY (id) )  [sqlalchemy.engine.Engine] 2026-06-14T09:35:00.141453Z [info ]  CREATE TABLE "OpenDataHubTestTable" ( id INTEGER NOT NULL, test_name VARCHAR(500) NOT NULL, start_time INTEGER NOT NULL, PRIMARY KEY (id) )  [sqlalchemy.engine.Engine] 2026-06-14T09:35:00.141612Z [info ] [no key 0.00005s] ()  [sqlalchemy.engine.Engine] 2026-06-14T09:35:00.141769Z [info ] [no key 0.00005s] ()  [sqlalchemy.engine.Engine] 2026-06-14T09:35:00.141928Z [info ] COMMIT  [sqlalchemy.engine.Engine] 2026-06-14T09:35:00.142279Z [info ] COMMIT  [sqlalchemy.engine.Engine] 2026-06-14T09:35:00.142538Z [info ] Trying to get client via new_client_from_config [ocp_resources.resource] 2026-06-14T09:35:00.149291Z [info ] kind: DataScienceCluster api version: datasciencecluster.opendatahub.io/v2 [ocp_resources.resource] /home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/urllib3/connectionpool.py:1110: InsecureRequestWarning: Unverified HTTPS request is being made to host 'api.ods-qe-psi-23.osp.rh-ods.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings warnings.warn( /home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/urllib3/connectionpool.py:1110: InsecureRequestWarning: Unverified HTTPS request is being made to host 'api.ods-qe-psi-23.osp.rh-ods.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings warnings.warn( /home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/urllib3/connectionpool.py:1110: InsecureRequestWarning: Unverified HTTPS request is being made to host 'api.ods-qe-psi-23.osp.rh-ods.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings warnings.warn( 2026-06-14T09:35:00.231942Z [info ] kind: DSCInitialization api version: dscinitialization.opendatahub.io/v2 [ocp_resources.resource] /home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/urllib3/connectionpool.py:1110: InsecureRequestWarning: Unverified HTTPS request is being made to host 'api.ods-qe-psi-23.osp.rh-ods.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings warnings.warn( ============================= test session starts ============================== platform linux -- Python 3.14.5, pytest-9.0.3, pluggy-1.6.0 -- /home/odh/opendatahub-tests/.venv/bin/python cachedir: .pytest_cache metadata: {'Python': '3.14.5', 'Platform': 'Linux-5.14.0-427.115.1.el9_4.x86_64-x86_64-with-glibc2.42', 'Packages': {'pytest': '9.0.3', 'pluggy': '1.6.0'}, 'Plugins': {'anyio': '4.13.0', 'langsmith': '0.8.9', 'asyncio': '1.4.0', 'dependency': '0.6.1', 'html': '4.2.0', 'metadata': '3.1.1', 'order': '1.4.0', 'progress': '1.4.0', 'testconfig': '0.2.0', 'xdist': '3.8.0', 'syrupy': '5.3.1'}} rootdir: /home/odh/opendatahub-tests configfile: pytest.ini plugins: anyio-4.13.0, langsmith-0.8.9, asyncio-1.4.0, dependency-0.6.1, html-4.2.0, metadata-3.1.1, order-1.4.0, progress-1.4.0, testconfig-0.2.0, xdist-3.8.0, syrupy-5.3.1 asyncio: mode=Mode.STRICT, debug=False, asyncio_default_fixture_loop_scope=None, asyncio_default_test_loop_scope=function collecting ... collected 35 items / 28 deselected / 7 selected tests/ogx/operator/test_ogx_server.py::TestOgxServer::test_ogx_server_verify_images[unprivileged_model_namespace0-ogx_server0]  ----------------------- test_ogx_server_verify_images[unprivileged_model_namespace0-ogx_server0] ----------------------- -------------------------------------------------------- SETUP -------------------------------------------------------- 2026-06-14T09:35:01.657048Z [info ] (OpenDataHubTestTable|OpenDataHubTestTable) _post_configure_properties() started [sqlalchemy.orm.mapper.Mapper] 2026-06-14T09:35:01.657254Z [info ] (OpenDataHubTestTable|OpenDataHubTestTable) initialize prop id [sqlalchemy.orm.mapper.Mapper] 2026-06-14T09:35:01.657440Z [info ] (OpenDataHubTestTable|OpenDataHubTestTable) initialize prop test_name [sqlalchemy.orm.mapper.Mapper] 2026-06-14T09:35:01.658008Z [info ] (OpenDataHubTestTable|OpenDataHubTestTable) initialize prop start_time [sqlalchemy.orm.mapper.Mapper] 2026-06-14T09:35:01.658450Z [info ] (OpenDataHubTestTable|OpenDataHubTestTable) _post_configure_properties() complete [sqlalchemy.orm.mapper.Mapper] 2026-06-14T09:35:01.659671Z [info ] BEGIN (implicit)  [sqlalchemy.engine.Engine] 2026-06-14T09:35:01.659851Z [info ] BEGIN (implicit)  [sqlalchemy.engine.Engine] 2026-06-14T09:35:01.660010Z [info ] INSERT INTO "OpenDataHubTestTable" (test_name, start_time) VALUES (?, ?) [sqlalchemy.engine.Engine] 2026-06-14T09:35:01.662336Z [info ] INSERT INTO "OpenDataHubTestTable" (test_name, start_time) VALUES (?, ?) [sqlalchemy.engine.Engine] 2026-06-14T09:35:01.662588Z [info ] [generated in 0.00013s] ('/home/odh/opendatahub-tests/tests/ogx/operator/test_ogx_server.py::test_ogx_server_verify_images[unprivileged_model_namespace0-ogx_server0]', 1781429701) [sqlalchemy.engine.Engine] 2026-06-14T09:35:01.662760Z [info ] [generated in 0.00013s] ('/home/odh/opendatahub-tests/tests/ogx/operator/test_ogx_server.py::test_ogx_server_verify_images[unprivileged_model_namespace0-ogx_server0]', 1781429701) [sqlalchemy.engine.Engine] 2026-06-14T09:35:01.662912Z [info ] COMMIT  [sqlalchemy.engine.Engine] 2026-06-14T09:35:01.666924Z [info ] COMMIT  [sqlalchemy.engine.Engine] 2026-06-14T09:35:01.667140Z [info ] Executing session fixture: event_loop_policy [conftest] 2026-06-14T09:35:01.667294Z [info ] Executing session fixture: _syrupy_apply_ide_patches [conftest] 2026-06-14T09:35:01.667428Z [info ] Executing session fixture: admin_client [conftest] 2026-06-14T09:35:01.667561Z [info ] Trying to get client via new_client_from_config [ocp_resources.resource] 2026-06-14T09:35:01.667697Z [info ] Executing session fixture: dsc_resource [conftest] 2026-06-14T09:35:01.667862Z [info ] kind: DataScienceCluster api version: datasciencecluster.opendatahub.io/v2 [ocp_resources.resource] 2026-06-14T09:35:01.727122Z [info ] Executing session fixture: tmp_path_factory [conftest] 2026-06-14T09:35:01.727270Z [info ] Executing session fixture: tests_tmp_dir [conftest] 2026-06-14T09:35:01.728248Z [info ] Executing session fixture: os_path_environment [conftest] 2026-06-14T09:35:01.728427Z [info ] Executing session fixture: tmpdir_factory [conftest] 2026-06-14T09:35:01.728580Z [info ] Executing session fixture: bin_directory [conftest] 2026-06-14T09:35:01.729606Z [info ] Executing session fixture: oc_binary_path [conftest] 2026-06-14T09:35:01.729814Z [info ] kind: ConsoleCLIDownload api version: console.openshift.io/v1 [ocp_resources.resource] 2026-06-14T09:35:01.756902Z [info ] All oc console cli download links: ['https://downloads-openshift-console.apps.ods-qe-psi-23.osp.rh-ods.com/amd64/linux/oc.tar'] [utilities.infra] name=utilities.infra 2026-06-14T09:35:01.757067Z [info ] Downloading archive using: url=https://downloads-openshift-console.apps.ods-qe-psi-23.osp.rh-ods.com/amd64/linux/oc.tar [utilities.infra] name=utilities.infra 2026-06-14T09:35:03.007064Z [info ] Extract the downloaded archive. [utilities.infra] name=utilities.infra 2026-06-14T09:35:03.307932Z [info ] Downloaded file: ['oc']  [utilities.infra] name=utilities.infra 2026-06-14T09:35:03.354251Z [info ] Executing session fixture: bin_directory_to_os_path [conftest] 2026-06-14T09:35:03.351469Z [info ] OC binary path: /tmp/pytest-VuB33eqQBAgJPsK3cBsZj5/bin0/oc [tests.conftest] name=tests.conftest 2026-06-14T09:35:03.351543Z [info ] Adding /tmp/pytest-VuB33eqQBAgJPsK3cBsZj5/bin0 to $PATH [tests.conftest] name=tests.conftest 2026-06-14T09:35:03.354681Z [info ] Executing session fixture: nodes [conftest] 2026-06-14T09:35:03.402643Z [info ] Executing session fixture: dsci_resource [conftest] 2026-06-14T09:35:03.402987Z [info ] kind: DSCInitialization api version: dscinitialization.opendatahub.io/v2 [ocp_resources.resource] 2026-06-14T09:35:03.417929Z [info ] Executing session fixture: record_testsuite_property [conftest] 2026-06-14T09:35:03.418287Z [info ] Executing session fixture: junitxml_plugin [conftest] 2026-06-14T09:35:03.418430Z [info ] Executing session fixture: cluster_sanity_scope_session [conftest] 2026-06-14T09:35:03.417077Z [info ] Check cluster sanity.  [utilities.infra] name=utilities.infra 2026-06-14T09:35:03.418749Z [info ] Verify all nodes are in a healthy condition. [ocp_utilities.infra] 2026-06-14T09:35:03.491562Z [info ] Verify all nodes are schedulable. [ocp_utilities.infra] 2026-06-14T09:35:03.562638Z [info ] Waiting for 120 seconds [0:02:00], retry every 5 seconds. (Function: utilities.infra.wait_for_dsci_status_ready Kwargs: {'dsci_resource': }) [timeout_sampler] 2026-06-14T09:35:03.561847Z [info ] Wait for DSCI default-dsci to be in Ready status. [utilities.infra] name=utilities.infra 2026-06-14T09:35:03.563279Z [info ] Get DSCInitialization default-dsci status [ocp_resources DSCInitialization] 2026-06-14T09:35:03.574318Z [info ] Elapsed time: 9.465217590332031e-05 [0:00:00.000095] [timeout_sampler] 2026-06-14T09:35:03.574473Z [info ] Waiting for 120 seconds [0:02:00], retry every 5 seconds. (Function: utilities.infra.wait_for_dsc_status_ready Kwargs: {'dsc_resource': }) [timeout_sampler] 2026-06-14T09:35:03.573568Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:35:03.574729Z [info ] Get DataScienceCluster default-dsc status [ocp_resources DataScienceCluster] 2026-06-14T09:35:08.606245Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:35:13.645925Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:35:18.685012Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:35:23.727505Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:35:28.766228Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:35:33.806529Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:35:38.849643Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:35:43.893787Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:35:48.932818Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:35:53.970899Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:35:59.010348Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:36:04.048742Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:36:09.086736Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:36:14.125681Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:36:19.157521Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:36:24.196072Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:36:29.236126Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:36:34.276976Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:36:39.316977Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:36:44.355731Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:36:49.397798Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:36:54.449164Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:36:59.490348Z [info ] Wait for DSC default-dsc are Ready. [utilities.infra] name=utilities.infra 2026-06-14T09:37:04.528645Z [error ] Cluster sanity check failed: Timed Out: 120 Function: utilities.infra.wait_for_dsc_status_ready Kwargs: {'dsc_resource': } Last exception: ResourceNotReadyError: DSC default-dsc is not ready. Current status: {'components': {'aipipelines': {'managementState': 'Removed'}, 'dashboard': {'managementState': 'Managed'}, 'feastoperator': {'managementState': 'Removed'}, 'kserve': {'managementState': 'Managed', 'releases': [{'name': 'KServe', 'repoUrl': 'https://github.com/kserve/kserve/', 'version': 'v0.17.0'}, {'name': 'vLLM', 'repoUrl': 'https://github.com/vllm-project/vllm', 'version': 'v0.18.0'}, {'name': 'llm-d-inference-scheduler', 'repoUrl': 'https://github.com/llm-d/llm-d-inference-scheduler', 'version': 'v0.7.1'}, {'name': 'llm-d-workload-variant-autoscaler', 'repoUrl': 'https://github.com/llm-d/llm-d-workload-variant-autoscaler', 'version': 'v0.6.0'}]}, 'kueue': {'managementState': 'Removed'}, 'llamastackoperator': {'managementState': 'Removed'}, 'mlflowoperator': {'managementState': 'Removed'}, 'modelregistry': {'managementState': 'Managed', 'registriesNamespace': 'rhoai-model-registries', 'releases': [{'name': 'Kubeflow Model Registry', 'repoUrl': 'https://github.com/kubeflow/model-registry', 'version': 'latest'}, {'name': 'Open Data Hub Model Registry Operator', 'repoUrl': 'https://github.com/opendatahub-io/model-registry-operator', 'version': 'latest'}]}, 'ray': {'managementState': 'Managed', 'releases': [{'name': 'KubeRay', 'repoUrl': 'https://github.com/ray-project/kuberay', 'version': '1.4.2'}]}, 'sparkoperator': {'managementState': 'Removed'}, 'trainer': {'managementState': 'Removed'}, 'trainingoperator': {'managementState': 'Managed', 'releases': [{'name': 'Kubeflow Training Operator', 'repoUrl': 'https://github.com/kubeflow/trainer', 'version': '1.9.0'}]}, 'trustyai': {'managementState': 'Managed', 'releases': [{'name': 'TrustyAI operator', 'repoUrl': 'https://github.com/trustyai-explainability/trustyai-service-operator', 'version': 'v1.37.0'}, {'name': 'TrustyAI service', 'repoUrl': 'https://github.com/trustyai-explainability/trustyai-explainability', 'version': 'v0.28.0'}, {'name': 'TrustyAI LMEval driver', 'repoUrl': 'https://github.com/trustyai-explainability/trustyai-service-operator', 'version': 'v1.37.0'}, {'name': 'TrustyAI LMEval job', 'repoUrl': 'https://github.com/EleutherAI/lm-evaluation-harness', 'version': 'v0.4.8'}, {'name': 'TrustyAI Guardrails orchestrator', 'repoUrl': 'https://github.com/foundation-model-stack/fms-guardrails-orchestrator', 'version': '0.9.4'}, {'name': 'TrustyAI builtin detectors', 'repoUrl': 'https://github.com/trustyai-explainability/guardrails-regex-detector', 'version': 'v0.2.0'}, {'name': 'TrustyAI sidecar gateway', 'repoUrl': 'https://github.com/trustyai-explainability/vllm-orchestrator-gateway', 'version': 'v0.2.1'}]}, 'workbenches': {'managementState': 'Managed', 'releases': [{'name': 'Kubeflow Notebook Controller', 'repoUrl': 'https://github.com/kubeflow/kubeflow', 'version': '1.10.0'}], 'workbenchNamespace': 'rhods-notebooks'}}, 'conditions': [{'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'failure deploying resource /default-kserve: apply failed ' 'components.platform.opendatahub.io/v1alpha1, Kind=Kserve: unable ' 'to patch components.platform.opendatahub.io/v1alpha1, Kind=Kserve ' 'default-kserve: Kserve.components.platform.opendatahub.io ' '"default-kserve" is invalid: [spec.modelsAsService: Invalid ' 'value: "null": spec.modelsAsService in body must be of type ' 'object: "null", : Invalid value: "null": some validation ' 'rules were not checked because the object was invalid; correct ' 'the existing errors to complete validation]', 'reason': 'Error', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'failure deploying resource /default-kserve: apply failed ' 'components.platform.opendatahub.io/v1alpha1, Kind=Kserve: unable ' 'to patch components.platform.opendatahub.io/v1alpha1, Kind=Kserve ' 'default-kserve: Kserve.components.platform.opendatahub.io ' '"default-kserve" is invalid: [spec.modelsAsService: Invalid ' 'value: "null": spec.modelsAsService in body must be of type ' 'object: "null", : Invalid value: "null": some validation ' 'rules were not checked because the object was invalid; correct ' 'the existing errors to complete validation]', 'observedGeneration': 3, 'reason': 'Error', 'status': 'False', 'type': 'ProvisioningSucceeded'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'ComponentsReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'AIPipelinesReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'DashboardReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'FeastOperatorReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'KserveLLMInferenceServiceDependencies'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'KserveLLMInferenceServiceWideEPDependencies'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'KserveReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'KueueReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'LlamaStackOperatorReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'MLflowOperatorReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'ModelControllerReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'ModelRegistryReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'ModelsAsServiceReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'RayReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'SparkOperatorReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'TrainerReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'TrainingOperatorReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'TrustyAIReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'WorkbenchesReady'}], 'observedGeneration': 1, 'phase': 'Not Ready', 'release': {'name': 'OpenShift AI Self-Managed', 'version': '3.4.0'}}. [utilities.infra] name=utilities.infra =============== 28 deselected, 11 warnings in 124.87s (0:02:04) ================ =============== 28 deselected, 11 warnings in 124.87s (0:02:04) ================ ! _pytest.outcomes.Exit: Cluster sanity check failed: Timed Out: 120 Function: utilities.infra.wait_for_dsc_status_ready Kwargs: {'dsc_resource': } Last exception: ResourceNotReadyError: DSC default-dsc is not ready. Current status: {'components': {'aipipelines': {'managementState': 'Removed'}, 'dashboard': {'managementState': 'Managed'}, 'feastoperator': {'managementState': 'Removed'}, 'kserve': {'managementState': 'Managed', 'releases': [{'name': 'KServe', 'repoUrl': 'https://github.com/kserve/kserve/', 'version': 'v0.17.0'}, {'name': 'vLLM', 'repoUrl': 'https://github.com/vllm-project/vllm', 'version': 'v0.18.0'}, {'name': 'llm-d-inference-scheduler', 'repoUrl': 'https://github.com/llm-d/llm-d-inference-scheduler', 'version': 'v0.7.1'}, {'name': 'llm-d-workload-variant-autoscaler', 'repoUrl': 'https://github.com/llm-d/llm-d-workload-variant-autoscaler', 'version': 'v0.6.0'}]}, 'kueue': {'managementState': 'Removed'}, 'llamastackoperator': {'managementState': 'Removed'}, 'mlflowoperator': {'managementState': 'Removed'}, 'modelregistry': {'managementState': 'Managed', 'registriesNamespace': 'rhoai-model-registries', 'releases': [{'name': 'Kubeflow Model Registry', 'repoUrl': 'https://github.com/kubeflow/model-registry', 'version': 'latest'}, {'name': 'Open Data Hub Model Registry Operator', 'repoUrl': 'https://github.com/opendatahub-io/model-registry-operator', 'version': 'latest'}]}, 'ray': {'managementState': 'Managed', 'releases': [{'name': 'KubeRay', 'repoUrl': 'https://github.com/ray-project/kuberay', 'version': '1.4.2'}]}, 'sparkoperator': {'managementState': 'Removed'}, 'trainer': {'managementState': 'Removed'}, 'trainingoperator': {'managementState': 'Managed', 'releases': [{'name': 'Kubeflow Training Operator', 'repoUrl': 'https://github.com/kubeflow/trainer', 'version': '1.9.0'}]}, 'trustyai': {'managementState': 'Managed', 'releases': [{'name': 'TrustyAI operator', 'repoUrl': 'https://github.com/trustyai-explainability/trustyai-service-operator', 'version': 'v1.37.0'}, {'name': 'TrustyAI service', 'repoUrl': 'https://github.com/trustyai-explainability/trustyai-explainability', 'version': 'v0.28.0'}, {'name': 'TrustyAI LMEval driver', 'repoUrl': 'https://github.com/trustyai-explainability/trustyai-service-operator', 'version': 'v1.37.0'}, {'name': 'TrustyAI LMEval job', 'repoUrl': 'https://github.com/EleutherAI/lm-evaluation-harness', 'version': 'v0.4.8'}, {'name': 'TrustyAI Guardrails orchestrator', 'repoUrl': 'https://github.com/foundation-model-stack/fms-guardrails-orchestrator', 'version': '0.9.4'}, {'name': 'TrustyAI builtin detectors', 'repoUrl': 'https://github.com/trustyai-explainability/guardrails-regex-detector', 'version': 'v0.2.0'}, {'name': 'TrustyAI sidecar gateway', 'repoUrl': 'https://github.com/trustyai-explainability/vllm-orchestrator-gateway', 'version': 'v0.2.1'}]}, 'workbenches': {'managementState': 'Managed', 'releases': [{'name': 'Kubeflow Notebook Controller', 'repoUrl': 'https://github.com/kubeflow/kubeflow', 'version': '1.10.0'}], 'workbenchNamespace': 'rhods-notebooks'}}, 'conditions': [{'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'failure deploying resource /default-kserve: apply failed ' 'components.platform.opendatahub.io/v1alpha1, Kind=Kserve: unable ' 'to patch components.platform.opendatahub.io/v1alpha1, Kind=Kserve ' 'default-kserve: Kserve.components.platform.opendatahub.io ' '"default-kserve" is invalid: [spec.modelsAsService: Invalid ' 'value: "null": spec.modelsAsService in body must be of type ' 'object: "null", : Invalid value: "null": some validation ' 'rules were not checked because the object was invalid; correct ' 'the existing errors to complete validation]', 'reason': 'Error', 'status': 'False', 'type': 'Ready'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'failure deploying resource /default-kserve: apply failed ' 'components.platform.opendatahub.io/v1alpha1, Kind=Kserve: unable ' 'to patch components.platform.opendatahub.io/v1alpha1, Kind=Kserve ' 'default-kserve: Kserve.components.platform.opendatahub.io ' '"default-kserve" is invalid: [spec.modelsAsService: Invalid ' 'value: "null": spec.modelsAsService in body must be of type ' 'object: "null", : Invalid value: "null": some validation ' 'rules were not checked because the object was invalid; correct ' 'the existing errors to complete validation]', 'observedGeneration': 3, 'reason': 'Error', 'status': 'False', 'type': 'ProvisioningSucceeded'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'ComponentsReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'AIPipelinesReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'DashboardReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'FeastOperatorReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'KserveLLMInferenceServiceDependencies'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'KserveLLMInferenceServiceWideEPDependencies'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'KserveReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'KueueReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'LlamaStackOperatorReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'MLflowOperatorReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'ModelControllerReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'ModelRegistryReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'ModelsAsServiceReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'RayReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'SparkOperatorReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'message': 'Component ManagementState is set to Removed', 'reason': 'Removed', 'severity': 'Info', 'status': 'False', 'type': 'TrainerReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'TrainingOperatorReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'TrustyAIReady'}, {'lastTransitionTime': '2026-06-14T09:23:58Z', 'status': 'True', 'type': 'WorkbenchesReady'}], 'observedGeneration': 1, 'phase': 'Not Ready', 'release': {'name': 'OpenShift AI Self-Managed', 'version': '3.4.0'}}. ! --- BVT pytest attempt 2026-06-14T09:37:06.977899+00:00 (python3 -m pytest...) --- ImportError while loading conftest '/home/odh/opendatahub-tests/conftest.py'. conftest.py:16: in from ocp_resources.cluster_service_version import ClusterServiceVersion E ModuleNotFoundError: No module named 'ocp_resources'