/workspace/source/test/e2e/tests/test_api_keys.py:1429: Cannot find maas-api pod in odh-ai-gateway-infra: error: error executing jsonpath "{.items[0].metadata.name}": Error executing template: array index out of bounds: index 0, length 0. Printing more information for debugging the template: template was: {.items[0].metadata.name} object given to jsonpath engine was: map[string]interface {}{"apiVersion":"v1", "items":[]interface {}{}, "kind":"List", "metadata":map[string]interface {}{"resourceVersion":""}}/workspace/source/test/e2e/tests/test_external_model_routing.py:481: provider withdrawal E2E for #108 is gated; set TENANT_EXTERNAL_MODEL_PARTIAL_WITHDRAWAL_E2E=true to enable (with it set, the #108 contract is expected to hold and a violation fails red)/workspace/source/test/e2e/tests/test_external_model_routing.py:564: provider withdrawal E2E for #106 is gated; set TENANT_EXTERNAL_MODEL_LAST_PROVIDER_WITHDRAWAL_E2E=true to enable (with it set, the #106 contract is expected to hold and a violation fails red)/workspace/source/test/e2e/tests/test_gateway_scoped_authpolicy.py:212: legacy gateway-default-auth is scoped to the shared default gateway/workspace/source/test/e2e/tests/test_namespace_scoping.py:214: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:287: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:324: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:382: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:456: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=trueself = <test_negative_security.TestMissingModelRef object at 0x7f06b0be8850> def test_authpolicy_with_nonexistent_model_ref(self): """MaaSAuthPolicy does not generate per-model AuthPolicies in gateway-only mode. Creates an auth policy referencing one valid model and one ghost model, waits for Degraded phase, then asserts that no legacy per-model AuthPolicies exist for either model. """ suffix = uuid.uuid4().hex[:8] policy_name = f"e2e-neg-ghost-policy-{suffix}" ghost_model = f"nonexistent-model-{suffix}" try: _create_test_auth_policy( policy_name, [MODEL_REF, ghost_model], groups=["system:authenticated"], ) > _wait_for_maas_auth_policy_phase(policy_name, "Degraded", require_auth_policies=False) test/e2e/tests/test_negative_security.py:578: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ name = 'e2e-neg-ghost-policy-7a883cd2', expected_phase = 'Degraded' namespace = 'models-as-a-service', timeout = 120, require_auth_policies = False require_enforced = True def _wait_for_maas_auth_policy_phase(name, expected_phase="Active", namespace=None, timeout=AUTHPOLICY_PHASE_TIMEOUT, require_auth_policies=False, require_enforced=True): """Wait for MaaSAuthPolicy to reach a specific phase. Args: name: Name of the MaaSAuthPolicy expected_phase: Phase to wait for (default: "Active") namespace: Namespace (defaults to _ns()) timeout: Maximum wait time in seconds (default: 60) require_auth_policies: If True, requires authPolicies to be populated (default: False). Keep False for gateway-only AuthPolicy reconciliation. require_enforced: If True (default): - with require_auth_policies=True: all status.authPolicies entries must be ready - with require_auth_policies=False and expected_phase Active: also wait for the gateway Kuadrant AuthPolicy (maas-gateway-auth) to be Accepted+Enforced Returns: The auth policy CR dict when the expected phase is reached Raises: TimeoutError: If MaaSAuthPolicy doesn't reach expected phase within timeout """ namespace = namespace or _ns() deadline = time.time() + timeout log.info(f"Waiting for MaaSAuthPolicy {name} to reach phase '{expected_phase}' (timeout: {timeout}s)...") last_snapshot = "not found" while time.time() < deadline: cr = _get_cr("maasauthpolicy", name, namespace) if cr is None: last_snapshot = "not found" else: status = cr.get("status", {}) phase = status.get("phase") auth_policies = status.get("authPolicies", []) ready_count = sum(1 for ap in auth_policies if ap.get("ready") is True) last_snapshot = f"phase={phase}, authPolicies={len(auth_policies)} (ready={ready_count})" if phase == expected_phase: # No per-model auth policies required — phase match is sufficient for the CR, # but gateway-only mode still needs Kuadrant Enforced before HTTP calls. # Callers that intentionally stop Kuadrant (e.g. TRLP degraded tests) must # pass require_enforced=False — Enforced cannot become True while Kuadrant is down. if not require_auth_policies: if require_enforced and expected_phase == "Active": # Keep a floor so a slow phase wait does not starve Kuadrant Enforced. # AuthConfig backlog after policy churn often needs the full enforced timeout. remaining = max(GATEWAY_ENFORCED_TIMEOUT, int(deadline - time.time())) _wait_for_gateway_auth_enforced(timeout=remaining) log.info(f"MaaSAuthPolicy {name} reached phase '{expected_phase}'") return cr # Auth policies required — check they exist if len(auth_policies) > 0: if require_enforced: all_enforced = all( ap.get("ready") is True for ap in auth_policies ) if all_enforced: log.info(f"MaaSAuthPolicy {name} reached phase '{expected_phase}' and all enforced") return cr else: log.info(f"MaaSAuthPolicy {name} reached phase '{expected_phase}' with {len(auth_policies)} auth policy status(es)") return cr log.debug(f"MaaSAuthPolicy {name}: {last_snapshot}") time.sleep(2) # Report what the loop last saw: a re-read after the deadline can show a # state the loop never evaluated. > raise TimeoutError( f"MaaSAuthPolicy {name} did not reach phase '{expected_phase}' within {timeout}s " f"(last seen: {last_snapshot})" ) E TimeoutError: MaaSAuthPolicy e2e-neg-ghost-policy-7a883cd2 did not reach phase 'Degraded' within 120s (last seen: phase=Pending, authPolicies=0 (ready=0)) test/e2e/tests/test_helper.py:1920: TimeoutError/workspace/source/test/e2e/tests/test_namespace_scoping.py:247: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_networkpolicy.py:119: AIGC praxis NetworkPolicy does not include OTLP 4317 egress (MaaS Go IPP bundle only)/workspace/source/test/e2e/tests/test_per_tenant_ipp_isolation.py:231: praxis-extproc; Go IPP GATEWAY_NAME/TENANT_NAMESPACE env not applicableself = <test_per_tenant_ipp_isolation.TestPerTenantIPPInfrastructure object at 0x7f06b0c16b10> ipp_tenant_cases = ({'suffix': 'f1222e', 'tenant_ns': 'ai-tenant-e2e-ipp-a-f1222e', 'tenant_label_name': 'e2e-ipp-a-f1222e', 'gateway_nam...t_ns': 'ai-tenant-e2e-ipp-b-e74c27', 'tenant_label_name': 'e2e-ipp-b-e74c27', 'gateway_name': 'e2e-ipp-b-e74c27', ...}) def test_multiple_tenant_ipp_stacks_coexist(self, ipp_tenant_cases): deployment_names = { per_tenant_ipp_names(case["tenant_label_name"])["processing_deployment"] for case in ipp_tenant_cases } deployment_names.add("payload-processing") assert len(deployment_names) == len(ipp_tenant_cases) + 1 for name in deployment_names: deployment = get_json_or_none("deployment", name, GATEWAY_NAMESPACE) > assert deployment is not None, f"missing IPP deployment {name}" E AssertionError: missing IPP deployment payload-processing-e2e-ipp-b-e74c27 E assert None is not None test/e2e/tests/test_per_tenant_ipp_isolation.py:300: AssertionError/workspace/source/test/e2e/tests/test_per_tenant_ipp_isolation.py:308: Per-tenant NetworkPolicy payload-processing-e2e-ipp-b-e74c27 not present (may be blocked by managed-ingress webhook on OpenShift)/workspace/source/test/e2e/tests/test_per_tenant_ipp_isolation.py:262: praxis-extproc; EnvoyFilter uses payload-*-extproc clusters (see test_per_tenant_praxis_isolation)cls = <class 'test_subscription.TestE2ESubscriptionFlow'> @classmethod def setup_class(cls): """Validate test environment prerequisites before running any tests. This validates that expected resources exist and are in the correct state. Tests will FAIL (not skip) if prerequisites are missing, ensuring CI catches issues. """ log.info("=" * 60) log.info("Validating E2E Test Prerequisites") log.info("=" * 60) # Validate MODEL_REF exists and is Ready model = _get_cr("maasmodelref", MODEL_REF, MODEL_NAMESPACE) if not model: pytest.fail(f"PREREQUISITE MISSING: MaaSModelRef '{MODEL_REF}' not found. " f"Ensure prow setup has created the model.") phase = model.get("status", {}).get("phase") endpoint = model.get("status", {}).get("endpoint") if phase != "Ready" or not endpoint: > pytest.fail(f"PREREQUISITE INVALID: MaaSModelRef '{MODEL_REF}' not Ready " f"(phase={phase}, endpoint={endpoint or 'none'}). " f"Wait for reconciliation or check controller logs.") E Failed: PREREQUISITE INVALID: MaaSModelRef 'simulator-w2-105228' not Ready (phase=Unhealthy, endpoint=none). Wait for reconciliation or check controller logs. test/e2e/tests/test_subscription.py:1981: Failedcls = <class 'test_subscription.TestE2ESubscriptionFlow'> @classmethod def setup_class(cls): """Validate test environment prerequisites before running any tests. This validates that expected resources exist and are in the correct state. Tests will FAIL (not skip) if prerequisites are missing, ensuring CI catches issues. """ log.info("=" * 60) log.info("Validating E2E Test Prerequisites") log.info("=" * 60) # Validate MODEL_REF exists and is Ready model = _get_cr("maasmodelref", MODEL_REF, MODEL_NAMESPACE) if not model: pytest.fail(f"PREREQUISITE MISSING: MaaSModelRef '{MODEL_REF}' not found. " f"Ensure prow setup has created the model.") phase = model.get("status", {}).get("phase") endpoint = model.get("status", {}).get("endpoint") if phase != "Ready" or not endpoint: > pytest.fail(f"PREREQUISITE INVALID: MaaSModelRef '{MODEL_REF}' not Ready " f"(phase={phase}, endpoint={endpoint or 'none'}). " f"Wait for reconciliation or check controller logs.") E Failed: PREREQUISITE INVALID: MaaSModelRef 'simulator-w2-105228' not Ready (phase=Unhealthy, endpoint=none). Wait for reconciliation or check controller logs. test/e2e/tests/test_subscription.py:1981: Failedcls = <class 'test_subscription.TestE2ESubscriptionFlow'> @classmethod def setup_class(cls): """Validate test environment prerequisites before running any tests. This validates that expected resources exist and are in the correct state. Tests will FAIL (not skip) if prerequisites are missing, ensuring CI catches issues. """ log.info("=" * 60) log.info("Validating E2E Test Prerequisites") log.info("=" * 60) # Validate MODEL_REF exists and is Ready model = _get_cr("maasmodelref", MODEL_REF, MODEL_NAMESPACE) if not model: pytest.fail(f"PREREQUISITE MISSING: MaaSModelRef '{MODEL_REF}' not found. " f"Ensure prow setup has created the model.") phase = model.get("status", {}).get("phase") endpoint = model.get("status", {}).get("endpoint") if phase != "Ready" or not endpoint: > pytest.fail(f"PREREQUISITE INVALID: MaaSModelRef '{MODEL_REF}' not Ready " f"(phase={phase}, endpoint={endpoint or 'none'}). " f"Wait for reconciliation or check controller logs.") E Failed: PREREQUISITE INVALID: MaaSModelRef 'simulator-w2-105228' not Ready (phase=Unhealthy, endpoint=none). Wait for reconciliation or check controller logs. test/e2e/tests/test_subscription.py:1981: Failedcls = <class 'test_subscription.TestE2ESubscriptionFlow'> @classmethod def setup_class(cls): """Validate test environment prerequisites before running any tests. This validates that expected resources exist and are in the correct state. Tests will FAIL (not skip) if prerequisites are missing, ensuring CI catches issues. """ log.info("=" * 60) log.info("Validating E2E Test Prerequisites") log.info("=" * 60) # Validate MODEL_REF exists and is Ready model = _get_cr("maasmodelref", MODEL_REF, MODEL_NAMESPACE) if not model: pytest.fail(f"PREREQUISITE MISSING: MaaSModelRef '{MODEL_REF}' not found. " f"Ensure prow setup has created the model.") phase = model.get("status", {}).get("phase") endpoint = model.get("status", {}).get("endpoint") if phase != "Ready" or not endpoint: > pytest.fail(f"PREREQUISITE INVALID: MaaSModelRef '{MODEL_REF}' not Ready " f"(phase={phase}, endpoint={endpoint or 'none'}). " f"Wait for reconciliation or check controller logs.") E Failed: PREREQUISITE INVALID: MaaSModelRef 'simulator-w2-105228' not Ready (phase=Unhealthy, endpoint=none). Wait for reconciliation or check controller logs. test/e2e/tests/test_subscription.py:1981: Failed