/workspace/source/test/e2e/tests/test_api_keys.py:930: CronJob maas-api-key-cleanup not found in opendatahub: Error from server (NotFound): cronjobs.batch "maas-api-key-cleanup" not found/workspace/source/test/e2e/tests/test_api_keys.py:977: NetworkPolicy maas-api-cleanup-restrict not found in opendatahub: Error from server (NotFound): networkpolicies.networking.k8s.io "maas-api-cleanup-restrict" not found/workspace/source/test/e2e/tests/test_api_keys.py:1096: Cannot find maas-api pod in opendatahub: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}self = <test_api_keys.TestAPIKeySubscriptionPhases object at 0x7f356c6f1580> def test_create_key_for_pending_subscription(self): """API key creation succeeds for Pending subscription.""" ns = _ns() subscription_name = "e2e-apikey-pending-sub" auth_name = "e2e-apikey-pending-auth" sa_name = "e2e-apikey-pending-sa" try: oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) _wait_reconcile(seconds=10) # Patch to Pending phase patch_data = { "status": { "phase": "Pending", "conditions": [{ "type": "Ready", "status": "False", "reason": "Pending", "message": "Reconciliation in progress", "lastTransitionTime": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") }], } } cmd = [ "kubectl", "patch", "maassubscription", subscription_name, "-n", ns, "--type=merge", "--subresource=status", "-p", json.dumps(patch_data) ] result = subprocess.run(cmd, capture_output=True, text=True) assert result.returncode == 0, f"Failed to patch: {result.stderr}" cr = _get_cr("maassubscription", subscription_name, namespace=ns) phase = cr.get("status", {}).get("phase") > assert phase == "Pending", f"Expected Pending, got {phase}" E AssertionError: Expected Pending, got Active E assert 'Active' == 'Pending' E E - Pending E + Active test/e2e/tests/test_api_keys.py:1337: AssertionError/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:320: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:377: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:453: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_subscription.py:1056: gateway-only mode: per-model AuthPolicy is not created/workspace/source/test/e2e/tests/test_tenant.py:127: Tenant not Active (e.g. Degraded); payload-processing not asserted/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:186: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deploy/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:358: Dormant-mode test mutates controller flags; set ENABLE_TENANT_DISCOVERY_DORMANT_E2E=true/workspace/source/test/e2e/tests/test_gateway_scoped_authpolicy.py:170: gateway-default-auth not present (maas-gateway-auth is active); scoping is validated by unit testsgateway_host = 'maas.apps.417ec702-7712-4adb-b215-7b4c45f5c3e3.prod.konfluxeaas.com' is_https = True @pytest.fixture(scope="session") def shared_test_tenants(gateway_host: str, is_https: bool): """ Shared tenant infrastructure for multi-tenant tests. Creates two AITenant instances (tenant-a, tenant-b) that persist for the entire test session. Tests requiring multiple tenants should use this fixture instead of creating their own tenants. Each tenant gets its own Gateway with a unique route hostname. Returns: tuple: (tenant_a_dict, tenant_b_dict) with keys: - name: tenant label name (e.g., "e2e-shared-a-abc123") - namespace: tenant namespace (e.g., "ai-tenant-e2e-shared-a-abc123") - base_url: maas-api URL for this tenant (derived from tenant's gateway route) - gateway_name: name of the Gateway CR for this tenant - suffix: 6-character hex suffix for unique resource names - policy_name: default auth policy name for this tenant - subscription_name: default subscription name for this tenant Requires: - AITenant CRD installed - Tenant namespace discovery enabled on maas-controller """ from multitenancy_helpers import ( require_aitenant_crd, new_named_tenant_case, bootstrap_aitenant_tenant, cleanup_discovery_case, wait_for_route_admitted, wait_for_deployment_available, ) require_aitenant_crd() # Create two persistent tenants for the session case_a = new_named_tenant_case("e2e-shared-a") case_b = new_named_tenant_case("e2e-shared-b") try: # Bootstrap both tenants (creates gateway + AITenant CR) for case in (case_a, case_b): bootstrap_aitenant_tenant(case) scheme = "https" for case in (case_a, case_b): route = wait_for_route_admitted(f"{case['gateway_name']}-route") try: host = route["spec"]["host"] except KeyError as e: raise RuntimeError( f"Route {case['gateway_name']}-route missing expected field: {e}. " f"Route structure: {route}" ) from e case["base_url"] = f"{scheme}://{host}/maas-api" # Wait for maas-api deployment to be ready before tests run # AITenant deployments are in MAAS_API_DEPLOYMENT_NAMESPACE, not tenant-specific namespace deployment_name = f"maas-api-{case['tenant_label_name']}" from test_helper import MAAS_API_DEPLOYMENT_NAMESPACE > wait_for_deployment_available(deployment_name, namespace=MAAS_API_DEPLOYMENT_NAMESPACE, timeout=180) test/e2e/tests/conftest.py:329: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:314: in wait_for_deployment_available return wait_for_json("deployment", name, namespace, predicate=_predicate, timeout=timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'deployment', name = 'maas-api-e2e-shared-a-866a5b' namespace = 'opendatahub' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: deployment/maas-api-e2e-shared-a-866a5b in opendatahub did not satisfy condition. Last object: None test/e2e/tests/multitenancy_helpers.py:227: AssertionErrorgateway_host = 'maas.apps.417ec702-7712-4adb-b215-7b4c45f5c3e3.prod.konfluxeaas.com' is_https = True @pytest.fixture(scope="session") def shared_test_tenants(gateway_host: str, is_https: bool): """ Shared tenant infrastructure for multi-tenant tests. Creates two AITenant instances (tenant-a, tenant-b) that persist for the entire test session. Tests requiring multiple tenants should use this fixture instead of creating their own tenants. Each tenant gets its own Gateway with a unique route hostname. Returns: tuple: (tenant_a_dict, tenant_b_dict) with keys: - name: tenant label name (e.g., "e2e-shared-a-abc123") - namespace: tenant namespace (e.g., "ai-tenant-e2e-shared-a-abc123") - base_url: maas-api URL for this tenant (derived from tenant's gateway route) - gateway_name: name of the Gateway CR for this tenant - suffix: 6-character hex suffix for unique resource names - policy_name: default auth policy name for this tenant - subscription_name: default subscription name for this tenant Requires: - AITenant CRD installed - Tenant namespace discovery enabled on maas-controller """ from multitenancy_helpers import ( require_aitenant_crd, new_named_tenant_case, bootstrap_aitenant_tenant, cleanup_discovery_case, wait_for_route_admitted, wait_for_deployment_available, ) require_aitenant_crd() # Create two persistent tenants for the session case_a = new_named_tenant_case("e2e-shared-a") case_b = new_named_tenant_case("e2e-shared-b") try: # Bootstrap both tenants (creates gateway + AITenant CR) for case in (case_a, case_b): bootstrap_aitenant_tenant(case) scheme = "https" for case in (case_a, case_b): route = wait_for_route_admitted(f"{case['gateway_name']}-route") try: host = route["spec"]["host"] except KeyError as e: raise RuntimeError( f"Route {case['gateway_name']}-route missing expected field: {e}. " f"Route structure: {route}" ) from e case["base_url"] = f"{scheme}://{host}/maas-api" # Wait for maas-api deployment to be ready before tests run # AITenant deployments are in MAAS_API_DEPLOYMENT_NAMESPACE, not tenant-specific namespace deployment_name = f"maas-api-{case['tenant_label_name']}" from test_helper import MAAS_API_DEPLOYMENT_NAMESPACE > wait_for_deployment_available(deployment_name, namespace=MAAS_API_DEPLOYMENT_NAMESPACE, timeout=180) test/e2e/tests/conftest.py:329: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:314: in wait_for_deployment_available return wait_for_json("deployment", name, namespace, predicate=_predicate, timeout=timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'deployment', name = 'maas-api-e2e-shared-a-866a5b' namespace = 'opendatahub' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: deployment/maas-api-e2e-shared-a-866a5b in opendatahub did not satisfy condition. Last object: None test/e2e/tests/multitenancy_helpers.py:227: AssertionErrorgateway_host = 'maas.apps.417ec702-7712-4adb-b215-7b4c45f5c3e3.prod.konfluxeaas.com' is_https = True @pytest.fixture(scope="session") def shared_test_tenants(gateway_host: str, is_https: bool): """ Shared tenant infrastructure for multi-tenant tests. Creates two AITenant instances (tenant-a, tenant-b) that persist for the entire test session. Tests requiring multiple tenants should use this fixture instead of creating their own tenants. Each tenant gets its own Gateway with a unique route hostname. Returns: tuple: (tenant_a_dict, tenant_b_dict) with keys: - name: tenant label name (e.g., "e2e-shared-a-abc123") - namespace: tenant namespace (e.g., "ai-tenant-e2e-shared-a-abc123") - base_url: maas-api URL for this tenant (derived from tenant's gateway route) - gateway_name: name of the Gateway CR for this tenant - suffix: 6-character hex suffix for unique resource names - policy_name: default auth policy name for this tenant - subscription_name: default subscription name for this tenant Requires: - AITenant CRD installed - Tenant namespace discovery enabled on maas-controller """ from multitenancy_helpers import ( require_aitenant_crd, new_named_tenant_case, bootstrap_aitenant_tenant, cleanup_discovery_case, wait_for_route_admitted, wait_for_deployment_available, ) require_aitenant_crd() # Create two persistent tenants for the session case_a = new_named_tenant_case("e2e-shared-a") case_b = new_named_tenant_case("e2e-shared-b") try: # Bootstrap both tenants (creates gateway + AITenant CR) for case in (case_a, case_b): bootstrap_aitenant_tenant(case) scheme = "https" for case in (case_a, case_b): route = wait_for_route_admitted(f"{case['gateway_name']}-route") try: host = route["spec"]["host"] except KeyError as e: raise RuntimeError( f"Route {case['gateway_name']}-route missing expected field: {e}. " f"Route structure: {route}" ) from e case["base_url"] = f"{scheme}://{host}/maas-api" # Wait for maas-api deployment to be ready before tests run # AITenant deployments are in MAAS_API_DEPLOYMENT_NAMESPACE, not tenant-specific namespace deployment_name = f"maas-api-{case['tenant_label_name']}" from test_helper import MAAS_API_DEPLOYMENT_NAMESPACE > wait_for_deployment_available(deployment_name, namespace=MAAS_API_DEPLOYMENT_NAMESPACE, timeout=180) test/e2e/tests/conftest.py:329: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:314: in wait_for_deployment_available return wait_for_json("deployment", name, namespace, predicate=_predicate, timeout=timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'deployment', name = 'maas-api-e2e-shared-a-866a5b' namespace = 'opendatahub' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: deployment/maas-api-e2e-shared-a-866a5b in opendatahub did not satisfy condition. Last object: None test/e2e/tests/multitenancy_helpers.py:227: AssertionErrorgateway_host = 'maas.apps.417ec702-7712-4adb-b215-7b4c45f5c3e3.prod.konfluxeaas.com' is_https = True @pytest.fixture(scope="session") def shared_test_tenants(gateway_host: str, is_https: bool): """ Shared tenant infrastructure for multi-tenant tests. Creates two AITenant instances (tenant-a, tenant-b) that persist for the entire test session. Tests requiring multiple tenants should use this fixture instead of creating their own tenants. Each tenant gets its own Gateway with a unique route hostname. Returns: tuple: (tenant_a_dict, tenant_b_dict) with keys: - name: tenant label name (e.g., "e2e-shared-a-abc123") - namespace: tenant namespace (e.g., "ai-tenant-e2e-shared-a-abc123") - base_url: maas-api URL for this tenant (derived from tenant's gateway route) - gateway_name: name of the Gateway CR for this tenant - suffix: 6-character hex suffix for unique resource names - policy_name: default auth policy name for this tenant - subscription_name: default subscription name for this tenant Requires: - AITenant CRD installed - Tenant namespace discovery enabled on maas-controller """ from multitenancy_helpers import ( require_aitenant_crd, new_named_tenant_case, bootstrap_aitenant_tenant, cleanup_discovery_case, wait_for_route_admitted, wait_for_deployment_available, ) require_aitenant_crd() # Create two persistent tenants for the session case_a = new_named_tenant_case("e2e-shared-a") case_b = new_named_tenant_case("e2e-shared-b") try: # Bootstrap both tenants (creates gateway + AITenant CR) for case in (case_a, case_b): bootstrap_aitenant_tenant(case) scheme = "https" for case in (case_a, case_b): route = wait_for_route_admitted(f"{case['gateway_name']}-route") try: host = route["spec"]["host"] except KeyError as e: raise RuntimeError( f"Route {case['gateway_name']}-route missing expected field: {e}. " f"Route structure: {route}" ) from e case["base_url"] = f"{scheme}://{host}/maas-api" # Wait for maas-api deployment to be ready before tests run # AITenant deployments are in MAAS_API_DEPLOYMENT_NAMESPACE, not tenant-specific namespace deployment_name = f"maas-api-{case['tenant_label_name']}" from test_helper import MAAS_API_DEPLOYMENT_NAMESPACE > wait_for_deployment_available(deployment_name, namespace=MAAS_API_DEPLOYMENT_NAMESPACE, timeout=180) test/e2e/tests/conftest.py:329: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/multitenancy_helpers.py:314: in wait_for_deployment_available return wait_for_json("deployment", name, namespace, predicate=_predicate, timeout=timeout) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'deployment', name = 'maas-api-e2e-shared-a-866a5b' namespace = 'opendatahub' def wait_for_json( kind: str, name: str, namespace: Optional[str] = None, *, predicate=None, timeout: int = 180, interval: int = 5, ) -> dict: deadline = time.time() + timeout last_obj = None while time.time() < deadline: obj = get_json_or_none(kind, name, namespace) if obj is not None: last_obj = obj if predicate is None or predicate(obj): return obj time.sleep(interval) > raise AssertionError( f"{kind}/{name} in {namespace or '<cluster>'} did not satisfy condition. Last object: {last_obj}" ) E AssertionError: deployment/maas-api-e2e-shared-a-866a5b in opendatahub did not satisfy condition. Last object: None test/e2e/tests/multitenancy_helpers.py:227: AssertionError