self = <test_tenant_health.TestTenantReconcileHealth object at 0x7f963927a910> def test_tenant_reaches_active_phase(self): """Tenant CR must reach Phase=Active with Ready=True. Fails immediately if Phase=Failed (e.g., RBAC forbidden error), surfacing the reconciler's error message in the test output. """ > tenant = _wait_for_tenant_phase("Active") test/e2e/tests/test_tenant_health.py:169: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ expected_phase = 'Active', timeout = 120 def _wait_for_tenant_phase(expected_phase, timeout=None): """Poll until Tenant reaches expected_phase. Fails fast on Phase=Failed. Returns the Tenant CR dict on success. Raises AssertionError on failure or timeout. """ timeout = timeout or TENANT_HEALTH_TIMEOUT deadline = time.time() + timeout log.info("Waiting for Tenant %s phase=%s (timeout: %ds)...", TENANT_NAME, expected_phase, timeout) while time.time() < deadline: tenant = _get_tenant() if tenant is None: log.debug("Tenant %s not found yet, retrying...", TENANT_NAME) time.sleep(3) continue phase = tenant.get("status", {}).get("phase", "") if phase == expected_phase: log.info("Tenant %s reached phase '%s'", TENANT_NAME, expected_phase) return tenant if phase == "Failed" and expected_phase != "Failed": ready = _get_condition(tenant, "Ready") reason = ready.get("reason", "unknown") if ready else "unknown" message = ready.get("message", "no message") if ready else "no message" pytest.fail( f"Tenant reconcile failed: {reason} — {message}" ) log.debug("Tenant %s: phase=%s, waiting for %s", TENANT_NAME, phase, expected_phase) time.sleep(3) tenant = _get_tenant() status = tenant.get("status", {}) if tenant else {} ready = _get_condition(tenant, "Ready") if tenant else None detail = f"reason={ready.get('reason')}, message={ready.get('message')}" if ready else "no Ready condition" > pytest.fail( f"Tenant {TENANT_NAME} did not reach phase '{expected_phase}' within {timeout}s " f"(current: phase={status.get('phase', 'none')}, {detail})" ) E Failed: Tenant default-tenant did not reach phase 'Active' within 120s (current: phase=Degraded, reason=Reconciled, message=MaaS platform manifests applied and maas-api deployment is available) test/e2e/tests/test_tenant_health.py:123: Failedself = <test_tenant_health.TestTenantReconcileHealth object at 0x7f963927a6d0> def test_tenant_conditions_healthy(self): """All Tenant status conditions must be in their healthy state.""" > tenant = _wait_for_tenant_phase("Active") test/e2e/tests/test_tenant_health.py:180: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ expected_phase = 'Active', timeout = 120 def _wait_for_tenant_phase(expected_phase, timeout=None): """Poll until Tenant reaches expected_phase. Fails fast on Phase=Failed. Returns the Tenant CR dict on success. Raises AssertionError on failure or timeout. """ timeout = timeout or TENANT_HEALTH_TIMEOUT deadline = time.time() + timeout log.info("Waiting for Tenant %s phase=%s (timeout: %ds)...", TENANT_NAME, expected_phase, timeout) while time.time() < deadline: tenant = _get_tenant() if tenant is None: log.debug("Tenant %s not found yet, retrying...", TENANT_NAME) time.sleep(3) continue phase = tenant.get("status", {}).get("phase", "") if phase == expected_phase: log.info("Tenant %s reached phase '%s'", TENANT_NAME, expected_phase) return tenant if phase == "Failed" and expected_phase != "Failed": ready = _get_condition(tenant, "Ready") reason = ready.get("reason", "unknown") if ready else "unknown" message = ready.get("message", "no message") if ready else "no message" pytest.fail( f"Tenant reconcile failed: {reason} — {message}" ) log.debug("Tenant %s: phase=%s, waiting for %s", TENANT_NAME, phase, expected_phase) time.sleep(3) tenant = _get_tenant() status = tenant.get("status", {}) if tenant else {} ready = _get_condition(tenant, "Ready") if tenant else None detail = f"reason={ready.get('reason')}, message={ready.get('message')}" if ready else "no Ready condition" > pytest.fail( f"Tenant {TENANT_NAME} did not reach phase '{expected_phase}' within {timeout}s " f"(current: phase={status.get('phase', 'none')}, {detail})" ) E Failed: Tenant default-tenant did not reach phase 'Active' within 120s (current: phase=Degraded, reason=Reconciled, message=MaaS platform manifests applied and maas-api deployment is available) test/e2e/tests/test_tenant_health.py:123: Failedself = <test_tenant_health.TestTenantManagedResources object at 0x7f963927a190> kind = 'deployment', min_count = 2 @pytest.mark.parametrize("kind,min_count", [ ("deployment", 2), ("service", 2), ("serviceaccount", 1), ("cronjob", 1), ("configmap", 1), ("networkpolicy", 1), ]) def test_core_resources_exist(self, kind, min_count): """Core Kubernetes resources must exist with tenant tracking labels.""" items = _list_resources_by_label(kind, namespace=DEPLOYMENT_NAMESPACE) assert items is not None, f"Resource type '{kind}' not available on cluster" names = [item["metadata"]["name"] for item in items] > assert len(items) >= min_count, ( f"Expected >= {min_count} {kind}(s) with label {TENANT_LABEL_SELECTOR} " f"in namespace {DEPLOYMENT_NAMESPACE}, found {len(items)}: {names}" ) E AssertionError: Expected >= 2 deployment(s) with label maas.opendatahub.io/tenant-name=default-tenant in namespace opendatahub, found 1: ['maas-api'] E assert 1 >= 2 E + where 1 = len([{'apiVersion': 'apps/v1', 'kind': 'Deployment', 'metadata': {'annotations': {'deployment.kubernetes.io/revision': '1'}, 'creationTimestamp': '2026-05-20T14:16:59Z', 'generation': 1, 'labels': {'app.kubernetes.io/component': 'api', 'app.kubernetes.io/name': 'maas-api', 'app.kubernetes.io/part-of': 'models-as-a-service', 'app.opendatahub.io/modelsasservice': 'true', ...}, ...}, 'spec': {'progressDeadlineSeconds': 600, 'replicas': 1, 'revisionHistoryLimit': 10, 'selector': {'matchLabels': {'app.kubernetes.io/component': 'api', 'app.kubernetes.io/name': 'maas-api', 'app.kubernetes.io/part-of': 'models-as-a-service'}}, ...}, ...}]) test/e2e/tests/test_tenant_health.py:220: AssertionErrorself = <test_tenant_health.TestTenantManagedResources object at 0x7f96392b6040> kind = 'service', min_count = 2 @pytest.mark.parametrize("kind,min_count", [ ("deployment", 2), ("service", 2), ("serviceaccount", 1), ("cronjob", 1), ("configmap", 1), ("networkpolicy", 1), ]) def test_core_resources_exist(self, kind, min_count): """Core Kubernetes resources must exist with tenant tracking labels.""" items = _list_resources_by_label(kind, namespace=DEPLOYMENT_NAMESPACE) assert items is not None, f"Resource type '{kind}' not available on cluster" names = [item["metadata"]["name"] for item in items] > assert len(items) >= min_count, ( f"Expected >= {min_count} {kind}(s) with label {TENANT_LABEL_SELECTOR} " f"in namespace {DEPLOYMENT_NAMESPACE}, found {len(items)}: {names}" ) E AssertionError: Expected >= 2 service(s) with label maas.opendatahub.io/tenant-name=default-tenant in namespace opendatahub, found 1: ['maas-api'] E assert 1 >= 2 E + where 1 = len([{'apiVersion': 'v1', 'kind': 'Service', 'metadata': {'annotations': {'service.alpha.openshift.io/serving-cert-signed-by': 'openshift-service-serving-signer@1779286090', 'service.beta.openshift.io/serving-cert-secret-name': 'maas-api-serving-cert', 'service.beta.openshift.io/serving-cert-signed-by': 'openshift-service-serving-signer@1779286090'}, 'creationTimestamp': '2026-05-20T14:16:59Z', 'labels': {'app.kubernetes.io/component': 'api', 'app.kubernetes.io/name': 'maas-api', 'app.kubernetes.io/part-of': 'models-as-a-service', 'app.opendatahub.io/modelsasservice': 'true', ...}, 'name': 'maas-api', ...}, 'spec': {'clusterIP': '172.31.209.213', 'clusterIPs': ['172.31.209.213'], 'internalTrafficPolicy': 'Cluster', 'ipFamilies': ['IPv4'], ...}, ...}]) test/e2e/tests/test_tenant_health.py:220: AssertionError/workspace/source/test/e2e/tests/test_tenant_health.py:272: CRD persesdashboards.perses.dev not registered on cluster/workspace/source/test/e2e/tests/test_tenant_health.py:272: CRD persesdatasources.perses.dev not registered on cluster