self = <test_api_keys.TestAPIKeySubscriptionFilter object at 0x7f4b9cfe1070> api_keys_base_url = 'https://maas.apps.0326b81c-bed8-4819-b607-79a904eecb0d.prod.konfluxeaas.com/maas-api/v1/api-keys' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6Imd4b3U2aUdQa3BGakdvQ2JTTV9DMDZEdTJ2Uk82X0ZyVkhDa1hRNEZ2SnMifQ.e...aW3foslHc2MqvbCGf19bS43KwgXa-7Lim3U4MI0JqO1jCXCzm5Ho2rR6ahZd3tpiXkE0s3DOOYTEBbzpQ', 'Content-Type': 'application/json'} def test_search_without_subscription_returns_all(self, api_keys_base_url: str, headers: dict): """Search without subscription filter returns keys across all subscriptions.""" key_ids = [] try: # Create keys with explicit subscription binding for i in range(2): r = requests.post( api_keys_base_url, headers=headers, json={"name": f"e2e-nofilter-{i}", "subscription": SIMULATOR_SUBSCRIPTION}, timeout=TIMEOUT, verify=TLS_VERIFY, ) > assert r.status_code in (200, 201), f"Failed to create key: {r.text}" E AssertionError: Failed to create key: E assert 403 in (200, 201) E + where 403 = <Response [403]>.status_code test/e2e/tests/test_api_keys.py:1576: AssertionError/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:320: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:377: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:453: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_subscription.py:1054: gateway-only mode: per-model AuthPolicy is not createdself = <test_aitenant_lifecycle.TestAITenantLifecycle object at 0x7f4b9cb18370> def test_aitenant_create_bootstrap_resources(self): case = _new_aitenant_case() try: _apply_gateway_fixture(case) _apply_aitenant(case) > _assert_aitenant_bootstrap_resources(case) test/e2e/tests/test_aitenant_lifecycle.py:423: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_aitenant_lifecycle.py:230: in _assert_aitenant_bootstrap_resources tenant_config = _wait_for_json( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maastenantconfig', name = 'default-tenant' namespace = 'ai-tenant-e2e-ait-3102a353' def _wait_for_json(kind, name, namespace=None, *, predicate=None, timeout=180, interval=5): deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = _get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError(f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}") E AssertionError: maastenantconfig/default-tenant in ai-tenant-e2e-ait-3102a353 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaasTenantConfig', 'metadata': {'annotations': {'maas.opendatahub.io/aitenant-name': 'e2e-ait-3102a353', 'maas.opendatahub.io/aitenant-namespace': 'ai-tenants'}, 'creationTimestamp': '2026-07-24T17:46:45Z', 'finalizers': ['maas.opendatahub.io/tenant-cleanup'], 'generation': 1, 'labels': {'ai-gateway.opendatahub.io/tenant': 'e2e-ait-3102a353', 'app.kubernetes.io/managed-by': 'maas-controller', 'app.kubernetes.io/part-of': 'modelsasservice', 'maas.opendatahub.io/managed-by-aitenant': 'true', 'maas.opendatahub.io/tenant-name': 'e2e-ait-3102a353', 'maas.opendatahub.io/tenant-namespace': 'ai-tenant-e2e-ait-3102a353'}, 'name': 'default-tenant', 'namespace': 'ai-tenant-e2e-ait-3102a353', 'resourceVersion': '44745', 'uid': '2b3160a0-9681-418d-9852-42f08afaa7b6'}, 'spec': {}, 'status': {'conditions': [{'lastTransitionTime': '2026-07-24T17:46:45Z', 'message': 'AuthConfig CRD (Kuadrant) is available on the cluster', 'observedGeneration': 1, 'reason': 'DependenciesMet', 'status': 'True', 'type': 'DependenciesAvailable'}, {'lastTransitionTime': '2026-07-24T17:46:45Z', 'message': 'Prerequisites satisfied; see Degraded for warnings', 'observedGeneration': 1, 'reason': 'PrerequisitesMet', 'status': 'True', 'type': 'MaaSPrerequisitesAvailable'}, {'lastTransitionTime': '2026-07-24T17:51:34Z', 'message': '', 'observedGeneration': 1, 'reason': 'Resolved', 'status': 'False', 'type': 'Degraded'}, {'lastTransitionTime': '2026-07-24T17:46:46Z', 'message': 'EnvoyFilter openshift-ingress/payload-processing targetRefs[0].name="maas-default-gateway"; expected gateway "e2e-ait-3102a353"', 'observedGeneration': 1, 'reason': 'DeploymentsNotReady', 'status': 'False', 'type': 'DeploymentsAvailable'}, {'lastTransitionTime': '2026-07-24T17:46:46Z', 'message': 'EnvoyFilter openshift-ingress/payload-processing targetRefs[0].name="maas-default-gateway"; expected gateway "e2e-ait-3102a353"', 'observedGeneration': 1, 'reason': 'DeploymentsNotReady', 'status': 'False', 'type': 'Ready'}], 'infraNamespace': 'odh-ai-gateway-infra', 'phase': 'Pending'}} test/e2e/tests/test_aitenant_lifecycle.py:114: AssertionErrorself = <test_aitenant_lifecycle.TestAITenantLifecycle object at 0x7f4b9d02cbb0> def test_aitenant_migrates_legacy_tenant_to_maas_tenant_config(self): if not _crd_exists(LEGACY_TENANT_CRD): pytest.skip(f"Missing CRD {LEGACY_TENANT_CRD}; legacy Tenant migration test is not applicable") suffix = uuid.uuid4().hex[:8] aitenant_name = f"e2e-migrate-{suffix}" tenant_ns = f"ai-tenant-{aitenant_name}" gateway_name = f"{aitenant_name}-gw" max_expiration_days = 37 try: _apply({"apiVersion": "v1", "kind": "Namespace", "metadata": {"name": tenant_ns}}) _apply_gateway_fixture({"gateway_name": gateway_name, "aitenant_name": aitenant_name}) _apply( { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "Tenant", "metadata": { "name": TENANT_NAME, "namespace": tenant_ns, }, "spec": { "gatewayRef": { "namespace": GATEWAY_NAMESPACE, "name": gateway_name, }, "apiKeys": { "maxExpirationDays": max_expiration_days, }, "externalOIDC": { "issuerUrl": "https://issuer.example.com/realms/e2e", "clientId": "e2e-client", "ttl": 600, }, "telemetry": { "enabled": False, "metrics": { "captureOrganization": False, "captureUser": True, "captureGroup": True, "captureModelUsage": False, }, }, }, } ) _apply_aitenant( { "aitenant_name": aitenant_name, "tenant_ns": tenant_ns, "gateway_name": gateway_name, } ) aitenant = _wait_for_json( AITENANT_KIND, aitenant_name, AITENANT_NAMESPACE, predicate=_aitenant_ready, timeout=240, ) assert aitenant["spec"]["gateway"]["name"] == gateway_name assert aitenant["spec"]["oidc"]["issuerUrl"] == "https://issuer.example.com/realms/e2e" assert aitenant["spec"]["oidc"]["clientId"] == "e2e-client" assert aitenant["status"]["gatewayRef"] == { "namespace": GATEWAY_NAMESPACE, "name": gateway_name, } def migrated_config(obj): labels = obj.get("metadata", {}).get("labels") or {} annotations = obj.get("metadata", {}).get("annotations") or {} spec = obj.get("spec") or {} telemetry = spec.get("telemetry") or {} metrics = telemetry.get("metrics") or {} return ( _maas_tenant_config_ready(obj) and labels.get("maas.opendatahub.io/managed-by-aitenant") == "true" and labels.get("ai-gateway.opendatahub.io/tenant") == aitenant_name and labels.get("maas.opendatahub.io/tenant-name") == aitenant_name and labels.get("maas.opendatahub.io/tenant-namespace") == tenant_ns and annotations.get(ANNOTATION_AITENANT_NAME) == aitenant_name and annotations.get(ANNOTATION_AITENANT_NAMESPACE) == AITENANT_NAMESPACE and (spec.get("apiKeys") or {}).get("maxExpirationDays") == max_expiration_days and telemetry.get("enabled") is False and metrics.get("captureOrganization") is False and metrics.get("captureUser") is True and metrics.get("captureGroup") is True and metrics.get("captureModelUsage") is False ) > tenant_config = _wait_for_json( TENANT_CONFIG_KIND, TENANT_NAME, tenant_ns, predicate=migrated_config, timeout=180, ) test/e2e/tests/test_aitenant_lifecycle.py:518: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maastenantconfig', name = 'default-tenant' namespace = 'ai-tenant-e2e-migrate-54f231e4' def _wait_for_json(kind, name, namespace=None, *, predicate=None, timeout=180, interval=5): deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = _get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError(f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}") E AssertionError: maastenantconfig/default-tenant in ai-tenant-e2e-migrate-54f231e4 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaasTenantConfig', 'metadata': {'annotations': {'maas.opendatahub.io/aitenant-name': 'e2e-migrate-54f231e4', 'maas.opendatahub.io/aitenant-namespace': 'ai-tenants'}, 'creationTimestamp': '2026-07-24T17:52:11Z', 'finalizers': ['maas.opendatahub.io/tenant-cleanup'], 'generation': 1, 'labels': {'ai-gateway.opendatahub.io/tenant': 'e2e-migrate-54f231e4', 'app.kubernetes.io/managed-by': 'maas-controller', 'app.kubernetes.io/part-of': 'modelsasservice', 'maas.opendatahub.io/managed-by-aitenant': 'true', 'maas.opendatahub.io/tenant-name': 'e2e-migrate-54f231e4', 'maas.opendatahub.io/tenant-namespace': 'ai-tenant-e2e-migrate-54f231e4'}, 'name': 'default-tenant', 'namespace': 'ai-tenant-e2e-migrate-54f231e4', 'resourceVersion': '48519', 'uid': 'f1c9c78b-dea5-4df9-a11a-5e5ed49e13a3'}, 'spec': {'apiKeys': {'maxExpirationDays': 37}, 'telemetry': {'enabled': False, 'metrics': {'captureGroup': True, 'captureModelUsage': False, 'captureOrganization': False, 'captureUser': True}}}, 'status': {'conditions': [{'lastTransitionTime': '2026-07-24T17:52:11Z', 'message': 'AuthConfig CRD (Kuadrant) is available on the cluster', 'observedGeneration': 1, 'reason': 'DependenciesMet', 'status': 'True', 'type': 'DependenciesAvailable'}, {'lastTransitionTime': '2026-07-24T17:52:11Z', 'message': 'Prerequisites satisfied; see Degraded for warnings', 'observedGeneration': 1, 'reason': 'PrerequisitesMet', 'status': 'True', 'type': 'MaaSPrerequisitesAvailable'}, {'lastTransitionTime': '2026-07-24T17:55:14Z', 'message': '', 'observedGeneration': 1, 'reason': 'Resolved', 'status': 'False', 'type': 'Degraded'}, {'lastTransitionTime': '2026-07-24T17:52:12Z', 'message': 'EnvoyFilter openshift-ingress/payload-processing targetRefs[0].name="maas-default-gateway"; expected gateway "e2e-migrate-54f231e4-gw"', 'observedGeneration': 1, 'reason': 'DeploymentsNotReady', 'status': 'False', 'type': 'DeploymentsAvailable'}, {'lastTransitionTime': '2026-07-24T17:52:12Z', 'message': 'EnvoyFilter openshift-ingress/payload-processing targetRefs[0].name="maas-default-gateway"; expected gateway "e2e-migrate-54f231e4-gw"', 'observedGeneration': 1, 'reason': 'DeploymentsNotReady', 'status': 'False', 'type': 'Ready'}], 'infraNamespace': 'odh-ai-gateway-infra', 'phase': 'Pending'}} test/e2e/tests/test_aitenant_lifecycle.py:114: AssertionErrorself = <test_aitenant_lifecycle.TestAITenantLifecycle object at 0x7f4b9d02ca90> def test_aitenant_delete_cleans_maas_resources_and_preserves_user_objects(self): case = _new_aitenant_case() try: _apply_gateway_fixture(case) _apply_aitenant(case) > _assert_aitenant_bootstrap_resources(case) test/e2e/tests/test_aitenant_lifecycle.py:553: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_aitenant_lifecycle.py:230: in _assert_aitenant_bootstrap_resources tenant_config = _wait_for_json( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maastenantconfig', name = 'default-tenant' namespace = 'ai-tenant-e2e-ait-20962f79' def _wait_for_json(kind, name, namespace=None, *, predicate=None, timeout=180, interval=5): deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = _get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError(f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}") E AssertionError: maastenantconfig/default-tenant in ai-tenant-e2e-ait-20962f79 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaasTenantConfig', 'metadata': {'annotations': {'maas.opendatahub.io/aitenant-name': 'e2e-ait-20962f79', 'maas.opendatahub.io/aitenant-namespace': 'ai-tenants'}, 'creationTimestamp': '2026-07-24T17:55:38Z', 'finalizers': ['maas.opendatahub.io/tenant-cleanup'], 'generation': 1, 'labels': {'ai-gateway.opendatahub.io/tenant': 'e2e-ait-20962f79', 'app.kubernetes.io/managed-by': 'maas-controller', 'app.kubernetes.io/part-of': 'modelsasservice', 'maas.opendatahub.io/managed-by-aitenant': 'true', 'maas.opendatahub.io/tenant-name': 'e2e-ait-20962f79', 'maas.opendatahub.io/tenant-namespace': 'ai-tenant-e2e-ait-20962f79'}, 'name': 'default-tenant', 'namespace': 'ai-tenant-e2e-ait-20962f79', 'resourceVersion': '52608', 'uid': 'b55ca10c-7977-4cd4-81cb-bdbbb71e2661'}, 'spec': {}, 'status': {'conditions': [{'lastTransitionTime': '2026-07-24T17:55:38Z', 'message': 'AuthConfig CRD (Kuadrant) is available on the cluster', 'observedGeneration': 1, 'reason': 'DependenciesMet', 'status': 'True', 'type': 'DependenciesAvailable'}, {'lastTransitionTime': '2026-07-24T17:55:38Z', 'message': 'Prerequisites satisfied; see Degraded for warnings', 'observedGeneration': 1, 'reason': 'PrerequisitesMet', 'status': 'True', 'type': 'MaaSPrerequisitesAvailable'}, {'lastTransitionTime': '2026-07-24T18:00:28Z', 'message': '', 'observedGeneration': 1, 'reason': 'Resolved', 'status': 'False', 'type': 'Degraded'}, {'lastTransitionTime': '2026-07-24T17:55:39Z', 'message': 'EnvoyFilter openshift-ingress/payload-processing targetRefs[0].name="maas-default-gateway"; expected gateway "e2e-ait-20962f79"', 'observedGeneration': 1, 'reason': 'DeploymentsNotReady', 'status': 'False', 'type': 'DeploymentsAvailable'}, {'lastTransitionTime': '2026-07-24T17:55:39Z', 'message': 'EnvoyFilter openshift-ingress/payload-processing targetRefs[0].name="maas-default-gateway"; expected gateway "e2e-ait-20962f79"', 'observedGeneration': 1, 'reason': 'DeploymentsNotReady', 'status': 'False', 'type': 'Ready'}], 'infraNamespace': 'odh-ai-gateway-infra', 'phase': 'Pending'}} test/e2e/tests/test_aitenant_lifecycle.py:114: AssertionErrorself = <test_aitenant_lifecycle.TestAITenantLifecycle object at 0x7f4b9d02c6d0> def test_aitenant_derives_non_default_tenant_namespace(self): """RHOAIENG-66836: non-default AITenant must not use models-as-a-service tenant namespace.""" suffix = uuid.uuid4().hex[:8] aitenant_name = f"e2e-derive-{suffix}" reserved_ns = os.environ.get("MAAS_SUBSCRIPTION_NAMESPACE", "models-as-a-service") expected_ns = f"ai-tenant-{aitenant_name}" gateway_name = aitenant_name try: _apply_gateway_fixture({"gateway_name": gateway_name, "aitenant_name": aitenant_name}) _apply( { "apiVersion": "maas.opendatahub.io/v1alpha1", "kind": "AITenant", "metadata": {"name": aitenant_name, "namespace": AITENANT_NAMESPACE}, "spec": {}, } ) aitenant = _wait_for_json( AITENANT_KIND, aitenant_name, AITENANT_NAMESPACE, predicate=_aitenant_ready, timeout=120, ) assert aitenant["status"]["tenantNamespace"] == expected_ns assert aitenant["status"]["tenantNamespace"] != reserved_ns > _wait_for_json( TENANT_CONFIG_KIND, TENANT_NAME, expected_ns, predicate=_maas_tenant_config_ready, timeout=300, ) test/e2e/tests/test_aitenant_lifecycle.py:630: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maastenantconfig', name = 'default-tenant' namespace = 'ai-tenant-e2e-derive-842768d6' def _wait_for_json(kind, name, namespace=None, *, predicate=None, timeout=180, interval=5): deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = _get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError(f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}") E AssertionError: maastenantconfig/default-tenant in ai-tenant-e2e-derive-842768d6 did not satisfy condition. Last object: {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaasTenantConfig', 'metadata': {'annotations': {'maas.opendatahub.io/aitenant-name': 'e2e-derive-842768d6', 'maas.opendatahub.io/aitenant-namespace': 'ai-tenants'}, 'creationTimestamp': '2026-07-24T18:01:03Z', 'finalizers': ['maas.opendatahub.io/tenant-cleanup'], 'generation': 1, 'labels': {'ai-gateway.opendatahub.io/tenant': 'e2e-derive-842768d6', 'app.kubernetes.io/managed-by': 'maas-controller', 'app.kubernetes.io/part-of': 'modelsasservice', 'maas.opendatahub.io/managed-by-aitenant': 'true', 'maas.opendatahub.io/tenant-name': 'e2e-derive-842768d6', 'maas.opendatahub.io/tenant-namespace': 'ai-tenant-e2e-derive-842768d6'}, 'name': 'default-tenant', 'namespace': 'ai-tenant-e2e-derive-842768d6', 'resourceVersion': '56532', 'uid': 'b4f36d43-b371-4730-8bc7-1bd769831ee5'}, 'spec': {}, 'status': {'conditions': [{'lastTransitionTime': '2026-07-24T18:01:03Z', 'message': 'AuthConfig CRD (Kuadrant) is available on the cluster', 'observedGeneration': 1, 'reason': 'DependenciesMet', 'status': 'True', 'type': 'DependenciesAvailable'}, {'lastTransitionTime': '2026-07-24T18:01:03Z', 'message': 'Prerequisites satisfied; see Degraded for warnings', 'observedGeneration': 1, 'reason': 'PrerequisitesMet', 'status': 'True', 'type': 'MaaSPrerequisitesAvailable'}, {'lastTransitionTime': '2026-07-24T18:05:57Z', 'message': '', 'observedGeneration': 1, 'reason': 'Resolved', 'status': 'False', 'type': 'Degraded'}, {'lastTransitionTime': '2026-07-24T18:01:04Z', 'message': 'EnvoyFilter openshift-ingress/payload-processing targetRefs[0].name="maas-default-gateway"; expected gateway "e2e-derive-842768d6"', 'observedGeneration': 1, 'reason': 'DeploymentsNotReady', 'status': 'False', 'type': 'DeploymentsAvailable'}, {'lastTransitionTime': '2026-07-24T18:01:04Z', 'message': 'EnvoyFilter openshift-ingress/payload-processing targetRefs[0].name="maas-default-gateway"; expected gateway "e2e-derive-842768d6"', 'observedGeneration': 1, 'reason': 'DeploymentsNotReady', 'status': 'False', 'type': 'Ready'}], 'infraNamespace': 'odh-ai-gateway-infra', 'phase': 'Pending'}} test/e2e/tests/test_aitenant_lifecycle.py:114: AssertionError