
--- BVT pytest attempt 2026-06-14T08:35:35.635545+00:00 (/.local/bin/uv run pytest...) ---
[2m2026-06-14T08:35:36.822581Z[0m [[32m[1minfo     [0m] [1mWriting tests log to /home/odh/opendatahub-tests/results/pytest-tests.log[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:36.832943Z[0m [[32m[1minfo     [0m] [1mBEGIN (implicit)              [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:36.833346Z[0m [[32m[1minfo     [0m] [1mBEGIN (implicit)              [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:36.833972Z[0m [[32m[1minfo     [0m] [1mPRAGMA main.table_info("OpenDataHubTestTable")[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:36.834116Z[0m [[32m[1minfo     [0m] [1mPRAGMA main.table_info("OpenDataHubTestTable")[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:36.834262Z[0m [[32m[1minfo     [0m] [1m[raw sql] ()                  [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:36.834562Z[0m [[32m[1minfo     [0m] [1m[raw sql] ()                  [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:36.834698Z[0m [[32m[1minfo     [0m] [1mPRAGMA temp.table_info("OpenDataHubTestTable")[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:36.834870Z[0m [[32m[1minfo     [0m] [1mPRAGMA temp.table_info("OpenDataHubTestTable")[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:36.835022Z[0m [[32m[1minfo     [0m] [1m[raw sql] ()                  [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:36.835168Z[0m [[32m[1minfo     [0m] [1m[raw sql] ()                  [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:36.835491Z[0m [[32m[1minfo     [0m] [1m
CREATE TABLE "OpenDataHubTestTable" (
	id INTEGER NOT NULL, 
	test_name VARCHAR(500) NOT NULL, 
	start_time INTEGER NOT NULL, 
	PRIMARY KEY (id)
)

[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:36.837748Z[0m [[32m[1minfo     [0m] [1m
CREATE TABLE "OpenDataHubTestTable" (
	id INTEGER NOT NULL, 
	test_name VARCHAR(500) NOT NULL, 
	start_time INTEGER NOT NULL, 
	PRIMARY KEY (id)
)

[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:36.838040Z[0m [[32m[1minfo     [0m] [1m[no key 0.00005s] ()          [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:36.838243Z[0m [[32m[1minfo     [0m] [1m[no key 0.00005s] ()          [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:36.838416Z[0m [[32m[1minfo     [0m] [1mCOMMIT                        [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:36.838588Z[0m [[32m[1minfo     [0m] [1mCOMMIT                        [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:36.838790Z[0m [[32m[1minfo     [0m] [1mTrying to get client via new_client_from_config[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T08:35:36.843259Z[0m [[32m[1minfo     [0m] [1mkind: DataScienceCluster api version: datasciencecluster.opendatahub.io/v2[0m [[0m[1m[34mocp_resources.resource[0m][0m
/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/urllib3/connectionpool.py:1097: InsecureRequestWarning: Unverified HTTPS request is being made to host 'api.ods-qe-psi-23.osp.rh-ods.com'. Adding certificate verification is strongly advised. See: https://urllib3.readthedocs.io/en/latest/advanced-usage.html#tls-warnings
  warnings.warn(
[2m2026-06-14T08:35:36.924944Z[0m [[32m[1minfo     [0m] [1mkind: DSCInitialization api version: dscinitialization.opendatahub.io/v2[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T08:35:36.951800Z[0m [[32m[1minfo     [0m] [1mRunning model registry tests against default namespace[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:36.952045Z[0m [[32m[1minfo     [0m] [1mkind: DataScienceCluster api version: datasciencecluster.opendatahub.io/v2[0m [[0m[1m[34mocp_resources.resource[0m][0m
============================= test session starts ==============================
platform linux -- Python 3.14.5, pytest-9.0.3, pluggy-1.6.0 -- /home/odh/opendatahub-tests/.venv/bin/python
cachedir: .pytest_cache
metadata: {'Python': '3.14.5', 'Platform': 'Linux-5.14.0-427.115.1.el9_4.x86_64-x86_64-with-glibc2.42', 'Packages': {'pytest': '9.0.3', 'pluggy': '1.6.0'}, 'Plugins': {'anyio': '4.13.0', 'asyncio': '1.3.0', 'dependency': '0.6.1', 'html': '4.2.0', 'metadata': '3.1.1', 'order': '1.3.0', 'progress': '1.4.0', 'testconfig': '0.2.0', 'xdist': '3.8.0', 'syrupy': '5.1.0'}}
rootdir: /home/odh/opendatahub-tests
configfile: pytest.ini
plugins: anyio-4.13.0, asyncio-1.3.0, dependency-0.6.1, html-4.2.0, metadata-3.1.1, order-1.3.0, progress-1.4.0, testconfig-0.2.0, xdist-3.8.0, syrupy-5.1.0
asyncio: mode=Mode.STRICT, debug=False, asyncio_default_fixture_loop_scope=None, asyncio_default_test_loop_scope=function
collecting ... collected 105 items / 94 deselected / 11 selected

tests/model_serving/maas_billing/maas_api_key/test_api_key_auth_policy_validation.py::TestAuthPolicyApiKeyValidation::test_auth_policy_callback_url_uses_correct_namespace [1m
--------------------------------- test_auth_policy_callback_url_uses_correct_namespace ---------------------------------[0m
[1m-------------------------------------------------------- SETUP --------------------------------------------------------[0m
[2m2026-06-14T08:35:37.238556Z[0m [[32m[1minfo     [0m] [1m(OpenDataHubTestTable|OpenDataHubTestTable) _post_configure_properties() started[0m [[0m[1m[34msqlalchemy.orm.mapper.Mapper[0m][0m
[2m2026-06-14T08:35:37.238744Z[0m [[32m[1minfo     [0m] [1m(OpenDataHubTestTable|OpenDataHubTestTable) initialize prop id[0m [[0m[1m[34msqlalchemy.orm.mapper.Mapper[0m][0m
[2m2026-06-14T08:35:37.239307Z[0m [[32m[1minfo     [0m] [1m(OpenDataHubTestTable|OpenDataHubTestTable) initialize prop test_name[0m [[0m[1m[34msqlalchemy.orm.mapper.Mapper[0m][0m
[2m2026-06-14T08:35:37.239771Z[0m [[32m[1minfo     [0m] [1m(OpenDataHubTestTable|OpenDataHubTestTable) initialize prop start_time[0m [[0m[1m[34msqlalchemy.orm.mapper.Mapper[0m][0m
[2m2026-06-14T08:35:37.240064Z[0m [[32m[1minfo     [0m] [1m(OpenDataHubTestTable|OpenDataHubTestTable) _post_configure_properties() complete[0m [[0m[1m[34msqlalchemy.orm.mapper.Mapper[0m][0m
[2m2026-06-14T08:35:37.240396Z[0m [[32m[1minfo     [0m] [1mBEGIN (implicit)              [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:37.240566Z[0m [[32m[1minfo     [0m] [1mBEGIN (implicit)              [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:37.240736Z[0m [[32m[1minfo     [0m] [1mINSERT INTO "OpenDataHubTestTable" (test_name, start_time) VALUES (?, ?)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:37.241559Z[0m [[32m[1minfo     [0m] [1mINSERT INTO "OpenDataHubTestTable" (test_name, start_time) VALUES (?, ?)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:37.243572Z[0m [[32m[1minfo     [0m] [1m[generated in 0.00012s] ('/home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_api_key_auth_policy_validation.py::test_auth_policy_callback_url_uses_correct_namespace', 1781426137)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:37.243754Z[0m [[32m[1minfo     [0m] [1m[generated in 0.00012s] ('/home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_api_key_auth_policy_validation.py::test_auth_policy_callback_url_uses_correct_namespace', 1781426137)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:37.243931Z[0m [[32m[1minfo     [0m] [1mCOMMIT                        [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:37.248408Z[0m [[32m[1minfo     [0m] [1mCOMMIT                        [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:35:37.248516Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: event_loop_policy[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:37.248621Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: _syrupy_apply_ide_patches[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:37.248722Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: admin_client[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:37.248854Z[0m [[32m[1minfo     [0m] [1mTrying to get client via new_client_from_config[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T08:35:37.248960Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: dsc_resource[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:37.249069Z[0m [[32m[1minfo     [0m] [1mkind: DataScienceCluster api version: datasciencecluster.opendatahub.io/v2[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T08:35:37.314031Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: tmp_path_factory[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:37.314178Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: tests_tmp_dir[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:37.315055Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: os_path_environment[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:37.315169Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: tmpdir_factory[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:37.315279Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: bin_directory[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:37.316368Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: oc_binary_path[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:37.316561Z[0m [[32m[1minfo     [0m] [1mkind: ConsoleCLIDownload api version: console.openshift.io/v1[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T08:35:37.341342Z[0m [[32m[1minfo     [0m] [1mAll oc console cli download links: ['https://downloads-openshift-console.apps.ods-qe-psi-23.osp.rh-ods.com/amd64/linux/oc.tar'][0m [[0m[1m[34mutilities.infra[0m][0m [36mname[0m=[35mutilities.infra[0m
[2m2026-06-14T08:35:37.341407Z[0m [[32m[1minfo     [0m] [1mDownloading archive using: url=https://downloads-openshift-console.apps.ods-qe-psi-23.osp.rh-ods.com/amd64/linux/oc.tar[0m [[0m[1m[34mutilities.infra[0m][0m [36mname[0m=[35mutilities.infra[0m
[2m2026-06-14T08:35:50.801670Z[0m [[32m[1minfo     [0m] [1mExtract the downloaded archive.[0m [[0m[1m[34mutilities.infra[0m][0m [36mname[0m=[35mutilities.infra[0m
[2m2026-06-14T08:35:51.003702Z[0m [[32m[1minfo     [0m] [1mDownloaded file: ['oc']       [0m [[0m[1m[34mutilities.infra[0m][0m [36mname[0m=[35mutilities.infra[0m
[2m2026-06-14T08:35:51.041681Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: bin_directory_to_os_path[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:51.037145Z[0m [[32m[1minfo     [0m] [1mOC binary path: /tmp/pytest-Mi6jTcbCtyAXoCicEPrnWm/bin0/oc[0m [[0m[1m[34mtests.conftest[0m][0m [36mname[0m=[35mtests.conftest[0m
[2m2026-06-14T08:35:51.037250Z[0m [[32m[1minfo     [0m] [1mAdding /tmp/pytest-Mi6jTcbCtyAXoCicEPrnWm/bin0 to $PATH[0m [[0m[1m[34mtests.conftest[0m][0m [36mname[0m=[35mtests.conftest[0m
[2m2026-06-14T08:35:51.042286Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: nodes[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:51.098326Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: dsci_resource[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:51.098629Z[0m [[32m[1minfo     [0m] [1mkind: DSCInitialization api version: dscinitialization.opendatahub.io/v2[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T08:35:51.111513Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: record_testsuite_property[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:51.111812Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: junitxml_plugin[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:51.111966Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: cluster_sanity_scope_session[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:51.110625Z[0m [[32m[1minfo     [0m] [1mCheck cluster sanity.         [0m [[0m[1m[34mutilities.infra[0m][0m [36mname[0m=[35mutilities.infra[0m
[2m2026-06-14T08:35:51.112231Z[0m [[32m[1minfo     [0m] [1mVerify all nodes are in a healthy condition.[0m [[0m[1m[34mocp_utilities.infra[0m][0m
[2m2026-06-14T08:35:51.207695Z[0m [[32m[1minfo     [0m] [1mVerify all nodes are schedulable.[0m [[0m[1m[34mocp_utilities.infra[0m][0m
[2m2026-06-14T08:35:51.287036Z[0m [[32m[1minfo     [0m] [1mWaiting for 120 seconds [0:02:00], retry every 5 seconds. (Function: utilities.infra.wait_for_dsci_status_ready  Kwargs: {'dsci_resource': <ocp_resources.dsc_initialization.DSCInitialization object at 0x7f2f56d27610>})[0m [[0m[1m[34mtimeout_sampler[0m][0m
[2m2026-06-14T08:35:51.286222Z[0m [[32m[1minfo     [0m] [1mWait for DSCI default-dsci to be in Ready status.[0m [[0m[1m[34mutilities.infra[0m][0m [36mname[0m=[35mutilities.infra[0m
[2m2026-06-14T08:35:51.287767Z[0m [[32m[1minfo     [0m] [1mGet DSCInitialization default-dsci status[0m [[0m[1m[34mocp_resources DSCInitialization[0m][0m
[2m2026-06-14T08:35:51.299842Z[0m [[32m[1minfo     [0m] [1mElapsed time: 0.00014400482177734375 [0:00:00.000144][0m [[0m[1m[34mtimeout_sampler[0m][0m
[2m2026-06-14T08:35:51.300080Z[0m [[32m[1minfo     [0m] [1mWaiting for 120 seconds [0:02:00], retry every 5 seconds. (Function: utilities.infra.wait_for_dsc_status_ready  Kwargs: {'dsc_resource': <ocp_resources.data_science_cluster.DataScienceCluster object at 0x7f2f56d26ad0>})[0m [[0m[1m[34mtimeout_sampler[0m][0m
[2m2026-06-14T08:35:51.299278Z[0m [[32m[1minfo     [0m] [1mWait for DSC default-dsc are Ready.[0m [[0m[1m[34mutilities.infra[0m][0m [36mname[0m=[35mutilities.infra[0m
[2m2026-06-14T08:35:51.300369Z[0m [[32m[1minfo     [0m] [1mGet DataScienceCluster default-dsc status[0m [[0m[1m[34mocp_resources DataScienceCluster[0m][0m
[2m2026-06-14T08:35:51.314321Z[0m [[32m[1minfo     [0m] [1mElapsed time: 3.457069396972656e-05 [0:00:00.000035][0m [[0m[1m[34mtimeout_sampler[0m][0m
[2m2026-06-14T08:35:51.314707Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: autouse_fixtures[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:51.314866Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: maas_gateway_api_hostname[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:51.314981Z[0m [[32m[1minfo     [0m] [1mkind: Ingress api version: config.openshift.io/v1[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T08:35:51.342612Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: maas_gateway_api[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:51.367737Z[0m [[32m[1minfo     [0m] [1mkind: Gateway api version: gateway.networking.k8s.io/v1[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T08:35:51.381140Z[0m [[32m[1minfo     [0m] [1mReusing existing gateway openshift-ingress/maas-default-gateway[0m [[0m[1m[34mconftest[0m][0m [36mname[0m=[35mconftest[0m
[2m2026-06-14T08:35:51.382379Z[0m [[32m[1minfo     [0m] [1mWait for Gateway/maas-default-gateway's 'Programmed' condition to be 'True'[0m [[0m[1m[34mocp_resources Gateway[0m][0m
[2m2026-06-14T08:35:51.382648Z[0m [[32m[1minfo     [0m] [1mWait until Gateway maas-default-gateway is created[0m [[0m[1m[34mocp_resources Gateway[0m][0m
[2m2026-06-14T08:35:51.382855Z[0m [[32m[1minfo     [0m] [1mWaiting for 300 seconds [0:05:00], retry every 1 seconds. (Function: ocp_resources.resource.wait.lambda: self.exists)[0m [[0m[1m[34mtimeout_sampler[0m][0m
[2m2026-06-14T08:35:51.395004Z[0m [[32m[1minfo     [0m] [1mElapsed time: 4.00543212890625e-05 [0:00:00.000040][0m [[0m[1m[34mtimeout_sampler[0m][0m
[2m2026-06-14T08:35:51.395155Z[0m [[32m[1minfo     [0m] [1mWaiting for 299.986825466156 seconds [0:04:59.986825], retry every 1 seconds. (Function: ocp_resources.resource.wait_for_condition.lambda: self.instance)[0m [[0m[1m[34mtimeout_sampler[0m][0m
[2m2026-06-14T08:35:51.407976Z[0m [[32m[1minfo     [0m] [1mElapsed time: 3.4332275390625e-05 [0:00:00.000034][0m [[0m[1m[34mtimeout_sampler[0m][0m
[2m2026-06-14T08:35:51.408133Z[0m [[32m[1minfo     [0m] [1mResourceEdit: Backing up old data[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T08:35:51.425688Z[0m [[32m[1minfo     [0m] [1mResourceEdits: Updating data for resource Gateway maas-default-gateway[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T08:35:51.426002Z[0m [[32m[1minfo     [0m] [1mUpdate Gateway maas-default-gateway:
{'metadata': {'annotations': {'security.opendatahub.io/authorino-tls-bootstrap': 'true'}, 'name': 'maas-default-gateway'}}[0m [[0m[1m[34mocp_resources Gateway[0m][0m
[2m2026-06-14T08:35:51.447585Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: pytestconfig[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:51.447866Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: use_unprivileged_client[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:51.447982Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: kubconfig_filepath[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:51.448091Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: cluster_authentication[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:51.448198Z[0m [[32m[1minfo     [0m] [1mkind: Authentication api version: config.openshift.io/v1[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T08:35:51.460965Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: is_byoidc[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:51.474630Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: non_admin_user_password[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:51.498417Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: unprivileged_client[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:51.790922Z[0m [[32m[1minfo     [0m] [1mRequesting token for user odh-user1 from https://keycloak.qe.rh-ods.com/realms/openshift/protocol/openid-connect/token[0m [[0m[1m[34mutilities.user_utils[0m][0m
[2m2026-06-14T08:35:52.128975Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: maas_subscription_namespace[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:52.139777Z[0m [[32m[1minfo     [0m] [1mNamespace models-as-a-service already exists, reusing it[0m [[0m[1m[34mconftest[0m][0m [36mname[0m=[35mconftest[0m
[2m2026-06-14T08:35:52.140927Z[0m [[32m[1minfo     [0m] [1mExecuting session fixture: maas_subscription_controller_enabled_latest[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:35:52.141051Z[0m [[32m[1minfo     [0m] [1mResourceEdit: Backing up old data --- [DuplicateFilter: Last log `kind: Authentication api version: config.openshift.io/v1` repeated 3 times][0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T08:35:52.155528Z[0m [[32m[1minfo     [0m] [1mResourceEdits: Updating data for resource DataScienceCluster default-dsc[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T08:35:52.155891Z[0m [[32m[1minfo     [0m] [1mUpdate DataScienceCluster default-dsc:
{'spec': {'components': {'kserve': {'modelsAsService': {'managementState': 'Managed'}}}}, 'metadata': {'name': 'default-dsc'}}[0m [[0m[1m[34mocp_resources DataScienceCluster[0m][0m
[2m2026-06-14T08:35:52.188701Z[0m [[32m[1minfo     [0m] [1mWait for DataScienceCluster/default-dsc's 'ModelsAsServiceReady' condition to be 'True'[0m [[0m[1m[34mocp_resources DataScienceCluster[0m][0m
[2m2026-06-14T08:35:52.188926Z[0m [[32m[1minfo     [0m] [1mWait until DataScienceCluster default-dsc is created[0m [[0m[1m[34mocp_resources DataScienceCluster[0m][0m
[2m2026-06-14T08:35:52.189062Z[0m [[32m[1minfo     [0m] [1mWaiting for 900 seconds [0:15:00], retry every 1 seconds. (Function: ocp_resources.resource.wait.lambda: self.exists)[0m [[0m[1m[34mtimeout_sampler[0m][0m
[2m2026-06-14T08:35:52.202172Z[0m [[32m[1minfo     [0m] [1mElapsed time: 3.62396240234375e-05 [0:00:00.000036][0m [[0m[1m[34mtimeout_sampler[0m][0m
[2m2026-06-14T08:35:52.202330Z[0m [[32m[1minfo     [0m] [1mWaiting for 899.9863579273224 seconds [0:14:59.986358], retry every 1 seconds. (Function: ocp_resources.resource.wait_for_condition.lambda: self.instance)[0m [[0m[1m[34mtimeout_sampler[0m][0m
[2m2026-06-14T08:50:52.403567Z[0m [[32m[1minfo     [0m] [1mResourceEdits: Restoring data for resource DataScienceCluster default-dsc[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T08:50:52.403914Z[0m [[32m[1minfo     [0m] [1mUpdate DataScienceCluster default-dsc:
{'spec': {'components': {'kserve': {'modelsAsService': None}}}, 'metadata': {'name': 'default-dsc'}}[0m [[0m[1m[34mocp_resources DataScienceCluster[0m][0m
ERROR[1m
TEST: TestAuthPolicyApiKeyValidation.test_auth_policy_callback_url_uses_correct_namespace [setup] STATUS: [0;31mERROR[0m[0m
[2m2026-06-14T08:50:52.466726Z[0m [[31m[1merror    [0m] [1mTraceback (most recent call last):
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 353, in from_call
    result: TResult | None = func()
                             ~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 245, in <lambda>
    lambda: runtest_hook(item=item, **kwds),
            ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__
    return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
           ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec
    return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
           ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall
    raise exception
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall
    teardown.throw(exception)
    ~~~~~~~~~~~~~~^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup
    return (yield)
            ^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall
    res = hook_impl.function(*args)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 165, in pytest_runtest_setup
    item.session._setupstate.setup(item)
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 523, in setup
    col.setup()
    ~~~~~~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1723, in setup
    self._request._fillfixtures()
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 707, in _fillfixtures
    item.funcargs[argname] = self.getfixturevalue(argname)
                             ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 539, in getfixturevalue
    fixturedef = self._get_active_fixturedef(argname)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 627, in _get_active_fixturedef
    fixturedef.execute(request=subrequest)
    ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1110, in execute
    result: FixtureValue = ihook.pytest_fixture_setup(
                           ~~~~~~~~~~~~~~~~~~~~~~~~~~^
        fixturedef=self, request=request
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__
    return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
           ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec
    return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
           ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall
    raise exception
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall
    teardown.throw(exception)
    ~~~~~~~~~~~~~~^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pytest_asyncio/plugin.py", line 728, in pytest_fixture_setup
    return (yield)
            ^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall
    teardown.throw(exception)
    ~~~~~~~~~~~~~~^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper
    return result.get_result()
           ~~~~~~~~~~~~~~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result
    raise exc.with_traceback(tb)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper
    res = yield
          ^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall
    teardown.throw(exception)
    ~~~~~~~~~~~~~~^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/setuponly.py", line 36, in pytest_fixture_setup
    return (yield)
            ^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall
    res = hook_impl.function(*args)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1202, in pytest_fixture_setup
    result = call_fixture_func(fixturefunc, request, kwargs)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 908, in call_fixture_func
    fixture_result = next(generator)
  File "/home/odh/opendatahub-tests/tests/model_serving/maas_billing/conftest.py", line 1100, in maas_subscription_controller_enabled_latest
    dsc_resource.wait_for_condition(
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
        condition="ModelsAsServiceReady",
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        status="True",
        ^^^^^^^^^^^^^^
        timeout=900,
        ^^^^^^^^^^^^
    )
    ^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/ocp_resources/resource.py", line 1315, in wait_for_condition
    for sample in TimeoutSampler(
                  ~~~~~~~~~~~~~~^
        wait_timeout=timeout_watcher.remaining_time(),
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        sleep=sleep_time,
        ^^^^^^^^^^^^^^^^^
        func=lambda: self.instance,
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^
    ):
    ^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/timeout_sampler/__init__.py", line 182, in __iter__
    raise TimeoutExpiredError(self._get_exception_log(exp=last_exp), last_exp=last_exp)
timeout_sampler.TimeoutExpiredError: Timed Out: 899.9863579273224
Function: ocp_resources.resource.wait_for_condition.lambda: self.instance
Last exception: N/A.[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:50:52.467000Z[0m [[32m[1minfo     [0m] [1mMust-gather collection is enabled for /home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_api_key_auth_policy_validation.py::test_auth_policy_callback_url_uses_correct_namespace.[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:50:52.467140Z[0m [[32m[1minfo     [0m] [1mBEGIN (implicit)              [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:50:52.467270Z[0m [[32m[1minfo     [0m] [1mBEGIN (implicit)              [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:50:52.467416Z[0m [[32m[1minfo     [0m] [1mSELECT "OpenDataHubTestTable".start_time AS "OpenDataHubTestTable_start_time" 
FROM "OpenDataHubTestTable" 
WHERE "OpenDataHubTestTable".test_name = ?
 LIMIT ? OFFSET ?[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:50:52.467546Z[0m [[32m[1minfo     [0m] [1mSELECT "OpenDataHubTestTable".start_time AS "OpenDataHubTestTable_start_time" 
FROM "OpenDataHubTestTable" 
WHERE "OpenDataHubTestTable".test_name = ?
 LIMIT ? OFFSET ?[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:50:52.467719Z[0m [[32m[1minfo     [0m] [1m[generated in 0.00036s] ('/home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_api_key_auth_policy_validation.py::test_auth_policy_callback_url_uses_correct_namespace', 1, 0)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:50:52.468526Z[0m [[32m[1minfo     [0m] [1m[generated in 0.00036s] ('/home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_api_key_auth_policy_validation.py::test_auth_policy_callback_url_uses_correct_namespace', 1, 0)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:50:52.471685Z[0m [[32m[1minfo     [0m] [1mROLLBACK                      [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:50:52.471930Z[0m [[32m[1minfo     [0m] [1mROLLBACK                      [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:50:52.472119Z[0m [[32m[1minfo     [0m] [1mTrying to get client via new_client_from_config[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T08:50:52.477443Z[0m [[32m[1minfo     [0m] [1mkind: Subscription api version: operators.coreos.com/v1alpha1[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T08:50:52.564397Z[0m [[32m[1minfo     [0m] [1mChecking subscription rhods-operator[0m [[0m[1m[34mutilities.infra[0m][0m [36mname[0m=[35mutilities.infra[0m
[2m2026-06-14T08:50:52.578578Z[0m [[32m[1minfo     [0m] [1mExpected CSV: rhods-operator.3.4.0[0m [[0m[1m[34mutilities.infra[0m][0m [36mname[0m=[35mutilities.infra[0m
[2m2026-06-14T08:50:52.581461Z[0m [[32m[1minfo     [0m] [1mTrying to get client via new_client_from_config[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T08:50:52.586243Z[0m [[32m[1minfo     [0m] [1mkind: ClusterServiceVersion api version: operators.coreos.com/v1alpha1[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T08:50:52.745053Z[0m [[32m[1minfo     [0m] [1mRunning oc adm must-gather --dest-dir=/home/odh/opendatahub-tests/results/must-gather-collected/model_serving/maas_billing/maas_api_key/test_api_key_auth_policy_validation/TestAuthPolicyApiKeyValidation/test_auth_policy_callback_url_uses_correct_namespace/pytest_exception_interact --since=915s --image=registry.redhat.io/rhoai/odh-must-gather-rhel9@sha256:3de5ff9eee7e9b8e6650f61dbae642db3d3ccb813afb804397dc60d18f150ea6 command[0m [[0m[1m[34mpyhelper_utils.shell[0m][0m

_ ERROR at setup of TestAuthPolicyApiKeyValidation.test_auth_policy_callback_url_uses_correct_namespace _
Traceback (most recent call last):
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 353, in from_call
    result: TResult | None = func()
                             ~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 245, in <lambda>
    lambda: runtest_hook(item=item, **kwds),
            ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__
    return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
           ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec
    return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
           ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall
    raise exception
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall
    teardown.throw(exception)
    ~~~~~~~~~~~~~~^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup
    return (yield)
            ^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall
    res = hook_impl.function(*args)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 165, in pytest_runtest_setup
    item.session._setupstate.setup(item)
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 523, in setup
    col.setup()
    ~~~~~~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1723, in setup
    self._request._fillfixtures()
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 707, in _fillfixtures
    item.funcargs[argname] = self.getfixturevalue(argname)
                             ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 539, in getfixturevalue
    fixturedef = self._get_active_fixturedef(argname)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 627, in _get_active_fixturedef
    fixturedef.execute(request=subrequest)
    ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1110, in execute
    result: FixtureValue = ihook.pytest_fixture_setup(
                           ~~~~~~~~~~~~~~~~~~~~~~~~~~^
        fixturedef=self, request=request
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__
    return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
           ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec
    return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
           ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall
    raise exception
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall
    teardown.throw(exception)
    ~~~~~~~~~~~~~~^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pytest_asyncio/plugin.py", line 728, in pytest_fixture_setup
    return (yield)
            ^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall
    teardown.throw(exception)
    ~~~~~~~~~~~~~~^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 53, in run_old_style_hookwrapper
    return result.get_result()
           ~~~~~~~~~~~~~~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_result.py", line 103, in get_result
    raise exc.with_traceback(tb)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 38, in run_old_style_hookwrapper
    res = yield
          ^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall
    teardown.throw(exception)
    ~~~~~~~~~~~~~~^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/setuponly.py", line 36, in pytest_fixture_setup
    return (yield)
            ^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall
    res = hook_impl.function(*args)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1202, in pytest_fixture_setup
    result = call_fixture_func(fixturefunc, request, kwargs)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 908, in call_fixture_func
    fixture_result = next(generator)
  File "/home/odh/opendatahub-tests/tests/model_serving/maas_billing/conftest.py", line 1100, in maas_subscription_controller_enabled_latest
    dsc_resource.wait_for_condition(
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
        condition="ModelsAsServiceReady",
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        status="True",
        ^^^^^^^^^^^^^^
        timeout=900,
        ^^^^^^^^^^^^
    )
    ^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/ocp_resources/resource.py", line 1315, in wait_for_condition
    for sample in TimeoutSampler(
                  ~~~~~~~~~~~~~~^
        wait_timeout=timeout_watcher.remaining_time(),
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        sleep=sleep_time,
        ^^^^^^^^^^^^^^^^^
        func=lambda: self.instance,
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^
    ):
    ^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/timeout_sampler/__init__.py", line 182, in __iter__
    raise TimeoutExpiredError(self._get_exception_log(exp=last_exp), last_exp=last_exp)
timeout_sampler.TimeoutExpiredError: Timed Out: 899.9863579273224
Function: ocp_resources.resource.wait_for_condition.lambda: self.instance
Last exception: N/A.
_ 1 of 11 completed, 0 Pass, 0 Fail, 0 Skip, 0 XPass, 0 XFail, 1 Error, 0 ReRun _

tests/model_serving/maas_billing/maas_api_key/test_api_key_auth_policy_validation.py::TestAuthPolicyApiKeyValidation::test_api_key_can_list_models[ocp_token_for_actor0] [2m2026-06-14T08:56:12.785532Z[0m [[32m[1minfo     [0m] [1m/home/odh/opendatahub-tests/results/must-gather-collected/model_serving/maas_billing/maas_api_key/test_api_key_auth_policy_validation/TestAuthPolicyApiKeyValidation/test_auth_policy_callback_url_uses_correct_namespace/pytest_exception_interact/mg-1781426137.zip is collected successfully[0m [[0m[1m[34mutilities.must_gather_collector[0m][0m [36mname[0m=[35mutilities.must_gather_collector[0m
[1m------------------------------------------------------- TEARDOWN -------------------------------------------------------[0m
[1m
---------------------------------- test_api_key_can_list_models[ocp_token_for_actor0] ----------------------------------[0m
[1m-------------------------------------------------------- SETUP --------------------------------------------------------[0m
[2m2026-06-14T08:56:12.795813Z[0m [[32m[1minfo     [0m] [1mBEGIN (implicit)              [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:12.796273Z[0m [[32m[1minfo     [0m] [1mBEGIN (implicit)              [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
ERROR[2m2026-06-14T08:56:12.801919Z[0m [[32m[1minfo     [0m] [1mINSERT INTO "OpenDataHubTestTable" (test_name, start_time) VALUES (?, ?)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:12.804215Z[0m [[32m[1minfo     [0m] [1mINSERT INTO "OpenDataHubTestTable" (test_name, start_time) VALUES (?, ?)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:12.804614Z[0m [[32m[1minfo     [0m] [1m[cached since 1236s ago] ('/home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_api_key_auth_policy_validation.py::test_api_key_can_list_models[ocp_token_for_actor0]', 1781427372)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:12.804761Z[0m [[32m[1minfo     [0m] [1m[cached since 1236s ago] ('/home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_api_key_auth_policy_validation.py::test_api_key_can_list_models[ocp_token_for_actor0]', 1781427372)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:12.804921Z[0m [[32m[1minfo     [0m] [1mCOMMIT                        [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:12.805045Z[0m [[32m[1minfo     [0m] [1mCOMMIT                        [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[1m
TEST: TestAuthPolicyApiKeyValidation.test_api_key_can_list_models[ocp_token_for_actor0] [setup] STATUS: [0;31mERROR[0m[0m
[2m2026-06-14T08:56:12.805402Z[0m [[31m[1merror    [0m] [1mTraceback (most recent call last):
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 353, in from_call
    result: TResult | None = func()
                             ~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 245, in <lambda>
    lambda: runtest_hook(item=item, **kwds),
            ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__
    return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
           ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec
    return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
           ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall
    raise exception
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall
    teardown.throw(exception)
    ~~~~~~~~~~~~~~^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup
    return (yield)
            ^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall
    res = hook_impl.function(*args)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 165, in pytest_runtest_setup
    item.session._setupstate.setup(item)
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 523, in setup
    col.setup()
    ~~~~~~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1723, in setup
    self._request._fillfixtures()
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 707, in _fillfixtures
    item.funcargs[argname] = self.getfixturevalue(argname)
                             ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 539, in getfixturevalue
    fixturedef = self._get_active_fixturedef(argname)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 627, in _get_active_fixturedef
    fixturedef.execute(request=subrequest)
    ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1091, in execute
    raise exc.with_traceback(exc_tb)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1202, in pytest_fixture_setup
    result = call_fixture_func(fixturefunc, request, kwargs)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 908, in call_fixture_func
    fixture_result = next(generator)
  File "/home/odh/opendatahub-tests/tests/model_serving/maas_billing/conftest.py", line 1100, in maas_subscription_controller_enabled_latest
    dsc_resource.wait_for_condition(
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
        condition="ModelsAsServiceReady",
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        status="True",
        ^^^^^^^^^^^^^^
        timeout=900,
        ^^^^^^^^^^^^
    )
    ^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/ocp_resources/resource.py", line 1315, in wait_for_condition
    for sample in TimeoutSampler(
                  ~~~~~~~~~~~~~~^
        wait_timeout=timeout_watcher.remaining_time(),
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        sleep=sleep_time,
        ^^^^^^^^^^^^^^^^^
        func=lambda: self.instance,
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^
    ):
    ^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/timeout_sampler/__init__.py", line 182, in __iter__
    raise TimeoutExpiredError(self._get_exception_log(exp=last_exp), last_exp=last_exp)
timeout_sampler.TimeoutExpiredError: Timed Out: 899.9863579273224
Function: ocp_resources.resource.wait_for_condition.lambda: self.instance
Last exception: N/A.[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:56:12.805566Z[0m [[32m[1minfo     [0m] [1mMust-gather collection is enabled for /home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_api_key_auth_policy_validation.py::test_api_key_can_list_models[ocp_token_for_actor0].[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:56:12.805719Z[0m [[32m[1minfo     [0m] [1mBEGIN (implicit)              [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:12.805887Z[0m [[32m[1minfo     [0m] [1mBEGIN (implicit)              [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:12.806024Z[0m [[32m[1minfo     [0m] [1mSELECT "OpenDataHubTestTable".start_time AS "OpenDataHubTestTable_start_time" 
FROM "OpenDataHubTestTable" 
WHERE "OpenDataHubTestTable".test_name = ?
 LIMIT ? OFFSET ?[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:12.806143Z[0m [[32m[1minfo     [0m] [1mSELECT "OpenDataHubTestTable".start_time AS "OpenDataHubTestTable_start_time" 
FROM "OpenDataHubTestTable" 
WHERE "OpenDataHubTestTable".test_name = ?
 LIMIT ? OFFSET ?[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:12.806297Z[0m [[32m[1minfo     [0m] [1m[cached since 320.3s ago] ('/home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_api_key_auth_policy_validation.py::test_api_key_can_list_models[ocp_token_for_actor0]', 1, 0)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:12.806440Z[0m [[32m[1minfo     [0m] [1m[cached since 320.3s ago] ('/home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_api_key_auth_policy_validation.py::test_api_key_can_list_models[ocp_token_for_actor0]', 1, 0)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:12.806584Z[0m [[32m[1minfo     [0m] [1mROLLBACK                      [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:12.806709Z[0m [[32m[1minfo     [0m] [1mROLLBACK                      [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:12.811258Z[0m [[32m[1minfo     [0m] [1mTrying to get client via new_client_from_config[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T08:56:12.869097Z[0m [[32m[1minfo     [0m] [1mChecking subscription rhods-operator[0m [[0m[1m[34mutilities.infra[0m][0m [36mname[0m=[35mutilities.infra[0m
[2m2026-06-14T08:56:12.881918Z[0m [[32m[1minfo     [0m] [1mExpected CSV: rhods-operator.3.4.0[0m [[0m[1m[34mutilities.infra[0m][0m [36mname[0m=[35mutilities.infra[0m
[2m2026-06-14T08:56:12.894335Z[0m [[32m[1minfo     [0m] [1mkind: ClusterServiceVersion api version: operators.coreos.com/v1alpha1[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T08:56:13.059264Z[0m [[32m[1minfo     [0m] [1mRunning oc adm must-gather --dest-dir=/home/odh/opendatahub-tests/results/must-gather-collected/model_serving/maas_billing/maas_api_key/test_api_key_auth_policy_validation/TestAuthPolicyApiKeyValidation/test_api_key_can_list_models_ocp_token_for_actor0_/pytest_exception_interact --since=300s --image=registry.redhat.io/rhoai/odh-must-gather-rhel9@sha256:3de5ff9eee7e9b8e6650f61dbae642db3d3ccb813afb804397dc60d18f150ea6 command[0m [[0m[1m[34mpyhelper_utils.shell[0m][0m
[2m2026-06-14T08:56:23.798145Z[0m [[32m[1minfo     [0m] [1m/home/odh/opendatahub-tests/results/must-gather-collected/model_serving/maas_billing/maas_api_key/test_api_key_auth_policy_validation/TestAuthPolicyApiKeyValidation/test_api_key_can_list_models_ocp_token_for_actor0_/pytest_exception_interact/mg-1781427372.zip is collected successfully[0m [[0m[1m[34mutilities.must_gather_collector[0m][0m [36mname[0m=[35mutilities.must_gather_collector[0m

_ ERROR at setup of TestAuthPolicyApiKeyValidation.test_api_key_can_list_models[ocp_token_for_actor0] _
Traceback (most recent call last):
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 353, in from_call
    result: TResult | None = func()
                             ~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 245, in <lambda>
    lambda: runtest_hook(item=item, **kwds),
            ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__
    return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
           ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec
    return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
           ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall
    raise exception
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall
    teardown.throw(exception)
    ~~~~~~~~~~~~~~^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup
    return (yield)
            ^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall
    res = hook_impl.function(*args)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 165, in pytest_runtest_setup
    item.session._setupstate.setup(item)
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 523, in setup
    col.setup()
    ~~~~~~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1723, in setup
    self._request._fillfixtures()
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 707, in _fillfixtures
    item.funcargs[argname] = self.getfixturevalue(argname)
                             ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 539, in getfixturevalue
    fixturedef = self._get_active_fixturedef(argname)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 627, in _get_active_fixturedef
    fixturedef.execute(request=subrequest)
    ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1091, in execute
    raise exc.with_traceback(exc_tb)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1202, in pytest_fixture_setup
    result = call_fixture_func(fixturefunc, request, kwargs)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 908, in call_fixture_func
    fixture_result = next(generator)
  File "/home/odh/opendatahub-tests/tests/model_serving/maas_billing/conftest.py", line 1100, in maas_subscription_controller_enabled_latest
    dsc_resource.wait_for_condition(
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
        condition="ModelsAsServiceReady",
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        status="True",
        ^^^^^^^^^^^^^^
        timeout=900,
        ^^^^^^^^^^^^
    )
    ^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/ocp_resources/resource.py", line 1315, in wait_for_condition
    for sample in TimeoutSampler(
                  ~~~~~~~~~~~~~~^
        wait_timeout=timeout_watcher.remaining_time(),
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        sleep=sleep_time,
        ^^^^^^^^^^^^^^^^^
        func=lambda: self.instance,
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^
    ):
    ^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/timeout_sampler/__init__.py", line 182, in __iter__
    raise TimeoutExpiredError(self._get_exception_log(exp=last_exp), last_exp=last_exp)
timeout_sampler.TimeoutExpiredError: Timed Out: 899.9863579273224
Function: ocp_resources.resource.wait_for_condition.lambda: self.instance
Last exception: N/A.
_ 2 of 11 completed, 0 Pass, 0 Fail, 0 Skip, 0 XPass, 0 XFail, 2 Error, 0 ReRun _

tests/model_serving/maas_billing/maas_api_key/test_api_key_crud.py::TestAPIKeyCRUD::test_create_api_key[ocp_token_for_actor0] [1m------------------------------------------------------- TEARDOWN -------------------------------------------------------[0m
[1m
-------------------------------------- test_create_api_key[ocp_token_for_actor0] --------------------------------------[0m
[1m-------------------------------------------------------- SETUP --------------------------------------------------------[0m
[2m2026-06-14T08:56:23.801477Z[0m [[32m[1minfo     [0m] [1mBEGIN (implicit)              [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:23.801608Z[0m [[32m[1minfo     [0m] [1mBEGIN (implicit)              [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:23.801734Z[0m [[32m[1minfo     [0m] [1mINSERT INTO "OpenDataHubTestTable" (test_name, start_time) VALUES (?, ?)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:23.801909Z[0m [[32m[1minfo     [0m] [1mINSERT INTO "OpenDataHubTestTable" (test_name, start_time) VALUES (?, ?)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:23.802062Z[0m [[32m[1minfo     [0m] [1m[cached since 1247s ago] ('/home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_api_key_crud.py::test_create_api_key[ocp_token_for_actor0]', 1781427383)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:23.802199Z[0m [[32m[1minfo     [0m] [1m[cached since 1247s ago] ('/home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_api_key_crud.py::test_create_api_key[ocp_token_for_actor0]', 1781427383)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:23.802574Z[0m [[32m[1minfo     [0m] [1mCOMMIT                        [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:23.802699Z[0m [[32m[1minfo     [0m] [1mCOMMIT                        [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
ERROR[1m
TEST: TestAPIKeyCRUD.test_create_api_key[ocp_token_for_actor0] [setup] STATUS: [0;31mERROR[0m[0m
[2m2026-06-14T08:56:23.806977Z[0m [[31m[1merror    [0m] [1mTraceback (most recent call last):
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 353, in from_call
    result: TResult | None = func()
                             ~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 245, in <lambda>
    lambda: runtest_hook(item=item, **kwds),
            ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__
    return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
           ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec
    return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
           ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall
    raise exception
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall
    teardown.throw(exception)
    ~~~~~~~~~~~~~~^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup
    return (yield)
            ^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall
    res = hook_impl.function(*args)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 165, in pytest_runtest_setup
    item.session._setupstate.setup(item)
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 523, in setup
    col.setup()
    ~~~~~~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1723, in setup
    self._request._fillfixtures()
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 707, in _fillfixtures
    item.funcargs[argname] = self.getfixturevalue(argname)
                             ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 539, in getfixturevalue
    fixturedef = self._get_active_fixturedef(argname)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 627, in _get_active_fixturedef
    fixturedef.execute(request=subrequest)
    ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1091, in execute
    raise exc.with_traceback(exc_tb)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1202, in pytest_fixture_setup
    result = call_fixture_func(fixturefunc, request, kwargs)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 908, in call_fixture_func
    fixture_result = next(generator)
  File "/home/odh/opendatahub-tests/tests/model_serving/maas_billing/conftest.py", line 1100, in maas_subscription_controller_enabled_latest
    dsc_resource.wait_for_condition(
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
        condition="ModelsAsServiceReady",
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        status="True",
        ^^^^^^^^^^^^^^
        timeout=900,
        ^^^^^^^^^^^^
    )
    ^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/ocp_resources/resource.py", line 1315, in wait_for_condition
    for sample in TimeoutSampler(
                  ~~~~~~~~~~~~~~^
        wait_timeout=timeout_watcher.remaining_time(),
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        sleep=sleep_time,
        ^^^^^^^^^^^^^^^^^
        func=lambda: self.instance,
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^
    ):
    ^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/timeout_sampler/__init__.py", line 182, in __iter__
    raise TimeoutExpiredError(self._get_exception_log(exp=last_exp), last_exp=last_exp)
timeout_sampler.TimeoutExpiredError: Timed Out: 899.9863579273224
Function: ocp_resources.resource.wait_for_condition.lambda: self.instance
Last exception: N/A.[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:56:23.808960Z[0m [[32m[1minfo     [0m] [1mMust-gather collection is enabled for /home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_api_key_crud.py::test_create_api_key[ocp_token_for_actor0].[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T08:56:23.809105Z[0m [[32m[1minfo     [0m] [1mBEGIN (implicit)              [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:23.809233Z[0m [[32m[1minfo     [0m] [1mBEGIN (implicit)              [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:23.809356Z[0m [[32m[1minfo     [0m] [1mSELECT "OpenDataHubTestTable".start_time AS "OpenDataHubTestTable_start_time" 
FROM "OpenDataHubTestTable" 
WHERE "OpenDataHubTestTable".test_name = ?
 LIMIT ? OFFSET ?[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:23.809495Z[0m [[32m[1minfo     [0m] [1mSELECT "OpenDataHubTestTable".start_time AS "OpenDataHubTestTable_start_time" 
FROM "OpenDataHubTestTable" 
WHERE "OpenDataHubTestTable".test_name = ?
 LIMIT ? OFFSET ?[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:23.809641Z[0m [[32m[1minfo     [0m] [1m[cached since 331.3s ago] ('/home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_api_key_crud.py::test_create_api_key[ocp_token_for_actor0]', 1, 0)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:23.809834Z[0m [[32m[1minfo     [0m] [1m[cached since 331.3s ago] ('/home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_api_key_crud.py::test_create_api_key[ocp_token_for_actor0]', 1, 0)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:23.809972Z[0m [[32m[1minfo     [0m] [1mROLLBACK                      [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:23.810093Z[0m [[32m[1minfo     [0m] [1mROLLBACK                      [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T08:56:23.810211Z[0m [[32m[1minfo     [0m] [1mTrying to get client via new_client_from_config[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T08:56:23.867014Z[0m [[32m[1minfo     [0m] [1mChecking subscription rhods-operator[0m [[0m[1m[34mutilities.infra[0m][0m [36mname[0m=[35mutilities.infra[0m
[2m2026-06-14T08:56:23.879625Z[0m [[32m[1minfo     [0m] [1mExpected CSV: rhods-operator.3.4.0[0m [[0m[1m[34mutilities.infra[0m][0m [36mname[0m=[35mutilities.infra[0m
[2m2026-06-14T08:56:23.886186Z[0m [[32m[1minfo     [0m] [1mkind: ClusterServiceVersion api version: operators.coreos.com/v1alpha1[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T08:56:24.027244Z[0m [[32m[1minfo     [0m] [1mRunning oc adm must-gather --dest-dir=/home/odh/opendatahub-tests/results/must-gather-collected/model_serving/maas_billing/maas_api_key/test_api_key_crud/TestAPIKeyCRUD/test_create_api_key_ocp_token_for_actor0_/pytest_exception_interact --since=300s --image=registry.redhat.io/rhoai/odh-must-gather-rhel9@sha256:3de5ff9eee7e9b8e6650f61dbae642db3d3ccb813afb804397dc60d18f150ea6 command[0m [[0m[1m[34mpyhelper_utils.shell[0m][0m
[2m2026-06-14T09:01:43.443619Z[0m [[32m[1minfo     [0m] [1m/home/odh/opendatahub-tests/results/must-gather-collected/model_serving/maas_billing/maas_api_key/test_api_key_crud/TestAPIKeyCRUD/test_create_api_key_ocp_token_for_actor0_/pytest_exception_interact/mg-1781427383.zip is collected successfully[0m [[0m[1m[34mutilities.must_gather_collector[0m][0m [36mname[0m=[35mutilities.must_gather_collector[0m
[1m------------------------------------------------------- TEARDOWN -------------------------------------------------------[0m

__ ERROR at setup of TestAPIKeyCRUD.test_create_api_key[ocp_token_for_actor0] __
Traceback (most recent call last):
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 353, in from_call
    result: TResult | None = func()
                             ~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 245, in <lambda>
    lambda: runtest_hook(item=item, **kwds),
            ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__
    return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
           ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec
    return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
           ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall
    raise exception
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall
    teardown.throw(exception)
    ~~~~~~~~~~~~~~^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup
    return (yield)
            ^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall
    res = hook_impl.function(*args)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 165, in pytest_runtest_setup
    item.session._setupstate.setup(item)
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 523, in setup
    col.setup()
    ~~~~~~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1723, in setup
    self._request._fillfixtures()
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 707, in _fillfixtures
    item.funcargs[argname] = self.getfixturevalue(argname)
                             ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 539, in getfixturevalue
    fixturedef = self._get_active_fixturedef(argname)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 627, in _get_active_fixturedef
    fixturedef.execute(request=subrequest)
    ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1091, in execute
    raise exc.with_traceback(exc_tb)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1202, in pytest_fixture_setup
    result = call_fixture_func(fixturefunc, request, kwargs)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 908, in call_fixture_func
    fixture_result = next(generator)
  File "/home/odh/opendatahub-tests/tests/model_serving/maas_billing/conftest.py", line 1100, in maas_subscription_controller_enabled_latest
    dsc_resource.wait_for_condition(
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
        condition="ModelsAsServiceReady",
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        status="True",
        ^^^^^^^^^^^^^^
        timeout=900,
        ^^^^^^^^^^^^
    )
    ^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/ocp_resources/resource.py", line 1315, in wait_for_condition
    for sample in TimeoutSampler(
                  ~~~~~~~~~~~~~~^
        wait_timeout=timeout_watcher.remaining_time(),
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        sleep=sleep_time,
        ^^^^^^^^^^^^^^^^^
        func=lambda: self.instance,
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^
    ):
    ^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/timeout_sampler/__init__.py", line 182, in __iter__
    raise TimeoutExpiredError(self._get_exception_log(exp=last_exp), last_exp=last_exp)
timeout_sampler.TimeoutExpiredError: Timed Out: 899.9863579273224
Function: ocp_resources.resource.wait_for_condition.lambda: self.instance
Last exception: N/A.
_ 3 of 11 completed, 0 Pass, 0 Fail, 0 Skip, 0 XPass, 0 XFail, 3 Error, 0 ReRun _

tests/model_serving/maas_billing/maas_api_key/test_api_key_crud.py::TestAPIKeyCRUD::test_revoke_api_key[ocp_token_for_actor0] [1m
-------------------------------------- test_revoke_api_key[ocp_token_for_actor0] --------------------------------------[0m
[1m-------------------------------------------------------- SETUP --------------------------------------------------------[0m
[2m2026-06-14T09:01:43.446788Z[0m [[32m[1minfo     [0m] [1mBEGIN (implicit)              [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:43.446959Z[0m [[32m[1minfo     [0m] [1mBEGIN (implicit)              [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:43.447311Z[0m [[32m[1minfo     [0m] [1mINSERT INTO "OpenDataHubTestTable" (test_name, start_time) VALUES (?, ?)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:43.447435Z[0m [[32m[1minfo     [0m] [1mINSERT INTO "OpenDataHubTestTable" (test_name, start_time) VALUES (?, ?)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:43.447584Z[0m [[32m[1minfo     [0m] [1m[cached since 1566s ago] ('/home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_api_key_crud.py::test_revoke_api_key[ocp_token_for_actor0]', 1781427703)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:43.447735Z[0m [[32m[1minfo     [0m] [1m[cached since 1566s ago] ('/home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_api_key_crud.py::test_revoke_api_key[ocp_token_for_actor0]', 1781427703)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:43.448773Z[0m [[32m[1minfo     [0m] [1mCOMMIT                        [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:43.448956Z[0m [[32m[1minfo     [0m] [1mCOMMIT                        [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
ERROR[1m
TEST: TestAPIKeyCRUD.test_revoke_api_key[ocp_token_for_actor0] [setup] STATUS: [0;31mERROR[0m[0m
[2m2026-06-14T09:01:43.452434Z[0m [[31m[1merror    [0m] [1mTraceback (most recent call last):
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 353, in from_call
    result: TResult | None = func()
                             ~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 245, in <lambda>
    lambda: runtest_hook(item=item, **kwds),
            ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__
    return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
           ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec
    return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
           ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall
    raise exception
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall
    teardown.throw(exception)
    ~~~~~~~~~~~~~~^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup
    return (yield)
            ^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall
    res = hook_impl.function(*args)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 165, in pytest_runtest_setup
    item.session._setupstate.setup(item)
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 523, in setup
    col.setup()
    ~~~~~~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1723, in setup
    self._request._fillfixtures()
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 707, in _fillfixtures
    item.funcargs[argname] = self.getfixturevalue(argname)
                             ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 539, in getfixturevalue
    fixturedef = self._get_active_fixturedef(argname)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 627, in _get_active_fixturedef
    fixturedef.execute(request=subrequest)
    ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1091, in execute
    raise exc.with_traceback(exc_tb)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1202, in pytest_fixture_setup
    result = call_fixture_func(fixturefunc, request, kwargs)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 908, in call_fixture_func
    fixture_result = next(generator)
  File "/home/odh/opendatahub-tests/tests/model_serving/maas_billing/conftest.py", line 1100, in maas_subscription_controller_enabled_latest
    dsc_resource.wait_for_condition(
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
        condition="ModelsAsServiceReady",
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        status="True",
        ^^^^^^^^^^^^^^
        timeout=900,
        ^^^^^^^^^^^^
    )
    ^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/ocp_resources/resource.py", line 1315, in wait_for_condition
    for sample in TimeoutSampler(
                  ~~~~~~~~~~~~~~^
        wait_timeout=timeout_watcher.remaining_time(),
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        sleep=sleep_time,
        ^^^^^^^^^^^^^^^^^
        func=lambda: self.instance,
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^
    ):
    ^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/timeout_sampler/__init__.py", line 182, in __iter__
    raise TimeoutExpiredError(self._get_exception_log(exp=last_exp), last_exp=last_exp)
timeout_sampler.TimeoutExpiredError: Timed Out: 899.9863579273224
Function: ocp_resources.resource.wait_for_condition.lambda: self.instance
Last exception: N/A.[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T09:01:43.452585Z[0m [[32m[1minfo     [0m] [1mMust-gather collection is enabled for /home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_api_key_crud.py::test_revoke_api_key[ocp_token_for_actor0].[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T09:01:43.452707Z[0m [[32m[1minfo     [0m] [1mBEGIN (implicit)              [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:43.452858Z[0m [[32m[1minfo     [0m] [1mBEGIN (implicit)              [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:43.454810Z[0m [[32m[1minfo     [0m] [1mSELECT "OpenDataHubTestTable".start_time AS "OpenDataHubTestTable_start_time" 
FROM "OpenDataHubTestTable" 
WHERE "OpenDataHubTestTable".test_name = ?
 LIMIT ? OFFSET ?[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:43.455013Z[0m [[32m[1minfo     [0m] [1mSELECT "OpenDataHubTestTable".start_time AS "OpenDataHubTestTable_start_time" 
FROM "OpenDataHubTestTable" 
WHERE "OpenDataHubTestTable".test_name = ?
 LIMIT ? OFFSET ?[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:43.455170Z[0m [[32m[1minfo     [0m] [1m[cached since 651s ago] ('/home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_api_key_crud.py::test_revoke_api_key[ocp_token_for_actor0]', 1, 0)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:43.455309Z[0m [[32m[1minfo     [0m] [1m[cached since 651s ago] ('/home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_api_key_crud.py::test_revoke_api_key[ocp_token_for_actor0]', 1, 0)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:43.455432Z[0m [[32m[1minfo     [0m] [1mROLLBACK                      [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:43.455550Z[0m [[32m[1minfo     [0m] [1mROLLBACK                      [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:43.455667Z[0m [[32m[1minfo     [0m] [1mTrying to get client via new_client_from_config[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T09:01:43.517442Z[0m [[32m[1minfo     [0m] [1mChecking subscription rhods-operator[0m [[0m[1m[34mutilities.infra[0m][0m [36mname[0m=[35mutilities.infra[0m
[2m2026-06-14T09:01:43.530491Z[0m [[32m[1minfo     [0m] [1mExpected CSV: rhods-operator.3.4.0[0m [[0m[1m[34mutilities.infra[0m][0m [36mname[0m=[35mutilities.infra[0m
[2m2026-06-14T09:01:43.542228Z[0m [[32m[1minfo     [0m] [1mkind: ClusterServiceVersion api version: operators.coreos.com/v1alpha1[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T09:01:43.701970Z[0m [[32m[1minfo     [0m] [1mRunning oc adm must-gather --dest-dir=/home/odh/opendatahub-tests/results/must-gather-collected/model_serving/maas_billing/maas_api_key/test_api_key_crud/TestAPIKeyCRUD/test_revoke_api_key_ocp_token_for_actor0_/pytest_exception_interact --since=300s --image=registry.redhat.io/rhoai/odh-must-gather-rhel9@sha256:3de5ff9eee7e9b8e6650f61dbae642db3d3ccb813afb804397dc60d18f150ea6 command[0m [[0m[1m[34mpyhelper_utils.shell[0m][0m
[2m2026-06-14T09:01:54.400665Z[0m [[32m[1minfo     [0m] [1m/home/odh/opendatahub-tests/results/must-gather-collected/model_serving/maas_billing/maas_api_key/test_api_key_crud/TestAPIKeyCRUD/test_revoke_api_key_ocp_token_for_actor0_/pytest_exception_interact/mg-1781427703.zip is collected successfully[0m [[0m[1m[34mutilities.must_gather_collector[0m][0m [36mname[0m=[35mutilities.must_gather_collector[0m

__ ERROR at setup of TestAPIKeyCRUD.test_revoke_api_key[ocp_token_for_actor0] __
Traceback (most recent call last):
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 353, in from_call
    result: TResult | None = func()
                             ~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 245, in <lambda>
    lambda: runtest_hook(item=item, **kwds),
            ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__
    return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
           ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec
    return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
           ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall
    raise exception
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall
    teardown.throw(exception)
    ~~~~~~~~~~~~~~^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup
    return (yield)
            ^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall
    res = hook_impl.function(*args)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 165, in pytest_runtest_setup
    item.session._setupstate.setup(item)
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 523, in setup
    col.setup()
    ~~~~~~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1723, in setup
    self._request._fillfixtures()
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 707, in _fillfixtures
    item.funcargs[argname] = self.getfixturevalue(argname)
                             ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 539, in getfixturevalue
    fixturedef = self._get_active_fixturedef(argname)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 627, in _get_active_fixturedef
    fixturedef.execute(request=subrequest)
    ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1091, in execute
    raise exc.with_traceback(exc_tb)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1202, in pytest_fixture_setup
    result = call_fixture_func(fixturefunc, request, kwargs)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 908, in call_fixture_func
    fixture_result = next(generator)
  File "/home/odh/opendatahub-tests/tests/model_serving/maas_billing/conftest.py", line 1100, in maas_subscription_controller_enabled_latest
    dsc_resource.wait_for_condition(
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
        condition="ModelsAsServiceReady",
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        status="True",
        ^^^^^^^^^^^^^^
        timeout=900,
        ^^^^^^^^^^^^
    )
    ^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/ocp_resources/resource.py", line 1315, in wait_for_condition
    for sample in TimeoutSampler(
                  ~~~~~~~~~~~~~~^
        wait_timeout=timeout_watcher.remaining_time(),
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        sleep=sleep_time,
        ^^^^^^^^^^^^^^^^^
        func=lambda: self.instance,
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^
    ):
    ^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/timeout_sampler/__init__.py", line 182, in __iter__
    raise TimeoutExpiredError(self._get_exception_log(exp=last_exp), last_exp=last_exp)
timeout_sampler.TimeoutExpiredError: Timed Out: 899.9863579273224
Function: ocp_resources.resource.wait_for_condition.lambda: self.instance
Last exception: N/A.
_ 4 of 11 completed, 0 Pass, 0 Fail, 0 Skip, 0 XPass, 0 XFail, 4 Error, 0 ReRun _

tests/model_serving/maas_billing/maas_api_key/test_gateway_deny_default.py::TestGatewayDenyByDefault::test_gateway_default_auth_in_gateway_namespace_and_accepted [1m------------------------------------------------------- TEARDOWN -------------------------------------------------------[0m
[1m
----------------------------- test_gateway_default_auth_in_gateway_namespace_and_accepted -----------------------------[0m
[1m-------------------------------------------------------- SETUP --------------------------------------------------------[0m
[2m2026-06-14T09:01:54.403593Z[0m [[32m[1minfo     [0m] [1mBEGIN (implicit)              [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:54.403730Z[0m [[32m[1minfo     [0m] [1mBEGIN (implicit)              [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:54.403883Z[0m [[32m[1minfo     [0m] [1mINSERT INTO "OpenDataHubTestTable" (test_name, start_time) VALUES (?, ?)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:54.403999Z[0m [[32m[1minfo     [0m] [1mINSERT INTO "OpenDataHubTestTable" (test_name, start_time) VALUES (?, ?)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:54.404146Z[0m [[32m[1minfo     [0m] [1m[cached since 1577s ago] ('/home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_gateway_deny_default.py::test_gateway_default_auth_in_gateway_namespace_and_accepted', 1781427714)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:54.404281Z[0m [[32m[1minfo     [0m] [1m[cached since 1577s ago] ('/home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_gateway_deny_default.py::test_gateway_default_auth_in_gateway_namespace_and_accepted', 1781427714)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:54.404667Z[0m [[32m[1minfo     [0m] [1mCOMMIT                        [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:54.405885Z[0m [[32m[1minfo     [0m] [1mCOMMIT                        [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
ERROR[1m
TEST: TestGatewayDenyByDefault.test_gateway_default_auth_in_gateway_namespace_and_accepted [setup] STATUS: [0;31mERROR[0m[0m
[2m2026-06-14T09:01:54.409454Z[0m [[31m[1merror    [0m] [1mTraceback (most recent call last):
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 353, in from_call
    result: TResult | None = func()
                             ~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 245, in <lambda>
    lambda: runtest_hook(item=item, **kwds),
            ~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_hooks.py", line 512, in __call__
    return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult)
           ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_manager.py", line 120, in _hookexec
    return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
           ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 167, in _multicall
    raise exception
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 139, in _multicall
    teardown.throw(exception)
    ~~~~~~~~~~~~~~^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/capture.py", line 895, in pytest_runtest_setup
    return (yield)
            ^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/pluggy/_callers.py", line 121, in _multicall
    res = hook_impl.function(*args)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 165, in pytest_runtest_setup
    item.session._setupstate.setup(item)
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/runner.py", line 523, in setup
    col.setup()
    ~~~~~~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/python.py", line 1723, in setup
    self._request._fillfixtures()
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 707, in _fillfixtures
    item.funcargs[argname] = self.getfixturevalue(argname)
                             ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 539, in getfixturevalue
    fixturedef = self._get_active_fixturedef(argname)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 627, in _get_active_fixturedef
    fixturedef.execute(request=subrequest)
    ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1091, in execute
    raise exc.with_traceback(exc_tb)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 1202, in pytest_fixture_setup
    result = call_fixture_func(fixturefunc, request, kwargs)
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/_pytest/fixtures.py", line 908, in call_fixture_func
    fixture_result = next(generator)
  File "/home/odh/opendatahub-tests/tests/model_serving/maas_billing/conftest.py", line 1100, in maas_subscription_controller_enabled_latest
    dsc_resource.wait_for_condition(
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
        condition="ModelsAsServiceReady",
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        status="True",
        ^^^^^^^^^^^^^^
        timeout=900,
        ^^^^^^^^^^^^
    )
    ^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/ocp_resources/resource.py", line 1315, in wait_for_condition
    for sample in TimeoutSampler(
                  ~~~~~~~~~~~~~~^
        wait_timeout=timeout_watcher.remaining_time(),
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
        sleep=sleep_time,
        ^^^^^^^^^^^^^^^^^
        func=lambda: self.instance,
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^
    ):
    ^
  File "/home/odh/opendatahub-tests/.venv/lib64/python3.14/site-packages/timeout_sampler/__init__.py", line 182, in __iter__
    raise TimeoutExpiredError(self._get_exception_log(exp=last_exp), last_exp=last_exp)
timeout_sampler.TimeoutExpiredError: Timed Out: 899.9863579273224
Function: ocp_resources.resource.wait_for_condition.lambda: self.instance
Last exception: N/A.[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T09:01:54.409632Z[0m [[32m[1minfo     [0m] [1mMust-gather collection is enabled for /home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_gateway_deny_default.py::test_gateway_default_auth_in_gateway_namespace_and_accepted.[0m [[0m[1m[34mconftest[0m][0m
[2m2026-06-14T09:01:54.409790Z[0m [[32m[1minfo     [0m] [1mBEGIN (implicit)              [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:54.410026Z[0m [[32m[1minfo     [0m] [1mBEGIN (implicit)              [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:54.410158Z[0m [[32m[1minfo     [0m] [1mSELECT "OpenDataHubTestTable".start_time AS "OpenDataHubTestTable_start_time" 
FROM "OpenDataHubTestTable" 
WHERE "OpenDataHubTestTable".test_name = ?
 LIMIT ? OFFSET ?[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:54.410279Z[0m [[32m[1minfo     [0m] [1mSELECT "OpenDataHubTestTable".start_time AS "OpenDataHubTestTable_start_time" 
FROM "OpenDataHubTestTable" 
WHERE "OpenDataHubTestTable".test_name = ?
 LIMIT ? OFFSET ?[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:54.410422Z[0m [[32m[1minfo     [0m] [1m[cached since 661.9s ago] ('/home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_gateway_deny_default.py::test_gateway_default_auth_in_gateway_namespace_and_accepted', 1, 0)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:54.412363Z[0m [[32m[1minfo     [0m] [1m[cached since 661.9s ago] ('/home/odh/opendatahub-tests/tests/model_serving/maas_billing/maas_api_key/test_gateway_deny_default.py::test_gateway_default_auth_in_gateway_namespace_and_accepted', 1, 0)[0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:54.416779Z[0m [[32m[1minfo     [0m] [1mROLLBACK                      [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:54.416959Z[0m [[32m[1minfo     [0m] [1mROLLBACK                      [0m [[0m[1m[34msqlalchemy.engine.Engine[0m][0m
[2m2026-06-14T09:01:54.417103Z[0m [[32m[1minfo     [0m] [1mTrying to get client via new_client_from_config[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T09:01:54.476356Z[0m [[32m[1minfo     [0m] [1mChecking subscription rhods-operator[0m [[0m[1m[34mutilities.infra[0m][0m [36mname[0m=[35mutilities.infra[0m
[2m2026-06-14T09:01:54.489123Z[0m [[32m[1minfo     [0m] [1mExpected CSV: rhods-operator.3.4.0[0m [[0m[1m[34mutilities.infra[0m][0m [36mname[0m=[35mutilities.infra[0m
[2m2026-06-14T09:01:54.495720Z[0m [[32m[1minfo     [0m] [1mkind: ClusterServiceVersion api version: operators.coreos.com/v1alpha1[0m [[0m[1m[34mocp_resources.resource[0m][0m
[2m2026-06-14T09:01:54.641633Z[0m [[32m[1minfo     [0m] [1mRunning oc adm must-gather --dest-dir=/home/odh/opendatahub-tests/results/must-gather-collected/model_serving/maas_billing/maas_api_key/test_gateway_deny_default/TestGatewayDenyByDefault/test_gateway_default_auth_in_gateway_namespace_and_accepted/pytest_exception_interact --since=300s --image=registry.redhat.io/rhoai/odh-must-gather-rhel9@sha256:3de5ff9eee7e9b8e6650f61dbae642db3d3ccb813afb804397dc60d18f150ea6 command[0m [[0m[1m[34mpyhelper_utils.shell[0m][0m