/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:320: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:377: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:453: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_subscription.py:1054: gateway-only mode: per-model AuthPolicy is not createdself = <test_subscription.TestStatusReporting object at 0x7f45ede24d60> def test_subscription_degraded_trlp_blocks_inference(self): """ Test: Degraded subscription with TRLP not ready blocks inference. This test verifies that when a subscription enters Degraded phase due to TokenRateLimitPolicy not being ready (e.g., Kuadrant controller down), inference requests are blocked with appropriate error to prevent rate limits from being bypassed. Uses pre-deployed e2e-trlp-test-simulated model to avoid TRLP sharing with concurrent tests. Test flow: 1. Scale down Kuadrant controller 2. Create subscription with valid model - TRLP created but not accepted 3. Wait for subscription to enter Degraded phase (TRLP ready=false) 4. Create API key and verify inference is blocked (403 Forbidden) 5. Scale Kuadrant controller back up 6. Wait for subscription to reach Active phase (TRLP ready=true) 7. Verify inference works (200 OK) """ ns = _ns() subscription_name = "e2e-trlp-degraded-sub" auth_name = "e2e-trlp-degraded-auth" sa_name = "e2e-trlp-degraded-sa" try: # Step 1: Scale down Kuadrant controller BEFORE creating subscription log.info("Step 1: Scaling down Kuadrant controller...") _scale_kuadrant_controller_down() # Step 2: Create auth policy and subscription log.info("Step 2: Creating subscription with Kuadrant controller down...") > sa_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) test/e2e/tests/test_subscription.py:1976: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ sa_name = 'e2e-trlp-degraded-sa', namespace = 'llm', duration = '10m' def _create_sa_token(sa_name, namespace=None, duration="10m"): namespace = namespace or _ns() sa_result = subprocess.run( ["oc", "create", "sa", sa_name, "-n", namespace], capture_output=True, text=True ) if sa_result.returncode != 0 and "already exists" not in sa_result.stderr: > raise RuntimeError(f"Failed to create SA {sa_name}: {sa_result.stderr}") E RuntimeError: Failed to create SA e2e-trlp-degraded-sa: error: failed to create serviceaccount: Post "https://a30efdf3f8db34039b12307a75377ea8-4892054d1f42a133.elb.us-east-1.amazonaws.com:6443/api/v1/namespaces/llm/serviceaccounts?fieldManager=kubectl-create&fieldValidation=Ignore": net/http: TLS handshake timeout test/e2e/tests/test_helper.py:179: RuntimeErrorself = <test_subscription.TestStatusReporting object at 0x7f45ed8fa1c0> def test_subscription_status_transitions_on_model_deletion(self): """ Test: MaaSSubscription transitions from Active to Degraded/Failed when model is deleted. Creates a subscription with a temporary model, verifies Active status, then deletes the model and verifies status transitions appropriately. """ ns = _ns() subscription_name = "e2e-status-transition-sub" auth_name = "e2e-status-transition-auth" model_name = "e2e-temp-model-status" sa_name = "e2e-status-transition-sa" try: _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) # Create a temporary model _create_test_maas_model(model_name, llmis_name=MODEL_REF, namespace=MODEL_NAMESPACE) _wait_reconcile() # Create auth policy and subscription for the model _create_test_auth_policy(auth_name, model_name, users=[sa_user]) _create_test_subscription(subscription_name, model_name, users=[sa_user]) _wait_for_maas_auth_policy_phase(auth_name) > _wait_for_maas_subscription_phase(subscription_name) test/e2e/tests/test_subscription.py:2122: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:819: in _wait_for_maas_subscription_phase cr = _get_cr("maassubscription", name, namespace) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maassubscription', name = 'e2e-status-transition-sub' namespace = 'models-as-a-service' def _get_cr(kind, name, namespace=None): """Get a CR as dict, or None if not found. Retries on transient errors. Returns None only when the resource genuinely does not exist (server NotFound). Raises RuntimeError for other failures (RBAC, missing CRD, transport errors that persist after retries) so callers can distinguish infrastructure issues from true absence. """ namespace = namespace or _ns() max_retries = 3 retry_delay = 2 for attempt in range(max_retries): result = subprocess.run(["oc", "get", kind, name, "-n", namespace, "-o", "json"], capture_output=True, text=True) if result.returncode == 0: return json.loads(result.stdout) if attempt < max_retries - 1 and _is_transient_kubectl_error(result.stderr): log.warning( f"Transient kubectl error getting {kind}/{name} (attempt {attempt + 1}/{max_retries}): {result.stderr.strip()}" ) time.sleep(retry_delay * (attempt + 1)) continue # Terminal failure — distinguish not-found from other errors if _is_not_found_error(result.stderr): return None log.error( f"Failed to get {kind}/{name} in namespace '{namespace}' after {attempt + 1} attempts. " f"Last error: {result.stderr.strip()}" ) > raise RuntimeError( f"Failed to get {kind}/{name} in namespace '{namespace}': {result.stderr.strip()}" ) E RuntimeError: Failed to get maassubscription/e2e-status-transition-sub in namespace 'models-as-a-service': Unable to connect to the server: net/http: TLS handshake timeout test/e2e/tests/test_helper.py:390: RuntimeErrorself = <test_subscription.TestDegradedSubscriptionFiltering object at 0x7f45eddeca00> def test_degraded_healthy_model_allows_inference(self): """ Test: Inference to healthy model in Degraded subscription succeeds. Setup: 1. Create subscription with 1 valid + 1 missing model 2. Controller sets phase=Degraded, modelRefStatuses shows mixed health Verify: - Subscription is Degraded with one ready=true, one ready=false - Inference to the valid model succeeds (200) """ ns = _ns() subscription_name = "e2e-degraded-healthy-inf" auth_name = "e2e-degraded-healthy-inf-auth" sa_name = "e2e-degraded-healthy-inf-sa" missing_model = "nonexistent-model-inf" try: > oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) test/e2e/tests/test_subscription.py:2203: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ sa_name = 'e2e-degraded-healthy-inf-sa', namespace = 'llm', duration = '10m' def _create_sa_token(sa_name, namespace=None, duration="10m"): namespace = namespace or _ns() sa_result = subprocess.run( ["oc", "create", "sa", sa_name, "-n", namespace], capture_output=True, text=True ) if sa_result.returncode != 0 and "already exists" not in sa_result.stderr: > raise RuntimeError(f"Failed to create SA {sa_name}: {sa_result.stderr}") E RuntimeError: Failed to create SA e2e-degraded-healthy-inf-sa: error: failed to create serviceaccount: Post "https://a30efdf3f8db34039b12307a75377ea8-4892054d1f42a133.elb.us-east-1.amazonaws.com:6443/api/v1/namespaces/llm/serviceaccounts?fieldManager=kubectl-create&fieldValidation=Ignore": http2: client connection lost test/e2e/tests/test_helper.py:179: RuntimeErrorself = <test_subscription.TestDegradedSubscriptionFiltering object at 0x7f45eddecf70> def test_failed_subscription_blocks_inference(self): """ Test: Failed subscription blocks inference via OPA rule. Setup: 1. Create subscription with valid model (starts Active) 2. Create API key 3. Manually patch subscription to Failed phase 4. Verify inference is rejected by OPA (403) Note: We use manual patching because naturally creating a Failed subscription requires only invalid models, which don't have routes (404 before OPA runs). """ ns = _ns() subscription_name = "e2e-failed-sub-inf" auth_name = "e2e-failed-sub-inf-auth" sa_name = "e2e-failed-sub-inf-sa" try: > oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) test/e2e/tests/test_subscription.py:2282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ sa_name = 'e2e-failed-sub-inf-sa', namespace = 'llm', duration = '10m' def _create_sa_token(sa_name, namespace=None, duration="10m"): namespace = namespace or _ns() sa_result = subprocess.run( ["oc", "create", "sa", sa_name, "-n", namespace], capture_output=True, text=True ) if sa_result.returncode != 0 and "already exists" not in sa_result.stderr: > raise RuntimeError(f"Failed to create SA {sa_name}: {sa_result.stderr}") E RuntimeError: Failed to create SA e2e-failed-sub-inf-sa: error: failed to create serviceaccount: Post "https://a30efdf3f8db34039b12307a75377ea8-4892054d1f42a133.elb.us-east-1.amazonaws.com:6443/api/v1/namespaces/llm/serviceaccounts?fieldManager=kubectl-create&fieldValidation=Ignore": net/http: TLS handshake timeout test/e2e/tests/test_helper.py:179: RuntimeErrorself = <test_subscription.TestDegradedSubscriptionFiltering object at 0x7f45eddec5b0> def test_models_endpoint_with_degraded_subscription_api_key(self): """ Test: /v1/models with API key bound to Degraded subscription. Verify behavior when querying models list with a Degraded subscription. Current implementation may succeed (showing valid models) or fail depending on selector implementation. """ ns = _ns() subscription_name = "e2e-degraded-models-apikey" auth_name = "e2e-degraded-models-apikey-auth" sa_name = "e2e-degraded-models-apikey-sa" missing_model = "nonexistent-model-apikey" try: > oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) test/e2e/tests/test_subscription.py:2393: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ sa_name = 'e2e-degraded-models-apikey-sa', namespace = 'llm', duration = '10m' def _create_sa_token(sa_name, namespace=None, duration="10m"): namespace = namespace or _ns() sa_result = subprocess.run( ["oc", "create", "sa", sa_name, "-n", namespace], capture_output=True, text=True ) if sa_result.returncode != 0 and "already exists" not in sa_result.stderr: > raise RuntimeError(f"Failed to create SA {sa_name}: {sa_result.stderr}") E RuntimeError: Failed to create SA e2e-degraded-models-apikey-sa: error: failed to create serviceaccount: Post "https://a30efdf3f8db34039b12307a75377ea8-4892054d1f42a133.elb.us-east-1.amazonaws.com:6443/api/v1/namespaces/llm/serviceaccounts?fieldManager=kubectl-create&fieldValidation=Ignore": dial tcp 100.56.45.196:6443: connect: connection refused test/e2e/tests/test_helper.py:179: RuntimeError