/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:320: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:377: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:453: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_subscription.py:1054: gateway-only mode: per-model AuthPolicy is not createdself = <test_subscription.TestStatusReporting object at 0x7f276e591580> def test_authpolicy_degraded_status_with_partial_models(self): """ Test: MaaSAuthPolicy shows Degraded phase when some models are valid, some missing. Creates an auth policy with one valid and one missing model ref and verifies: - Phase is "Degraded" - authPolicies contains entry for the valid model (ready=true) """ ns = _ns() auth_name = "e2e-status-degraded-auth" sa_name = "e2e-status-degraded-auth-sa" missing_model = "nonexistent-model-auth-partial" try: _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) # Create auth policy with both valid and missing models _create_test_auth_policy(auth_name, [MODEL_REF, missing_model], users=[sa_user]) # Wait for auth policy to reach Degraded phase with polling > cr = _wait_for_maas_auth_policy_phase(auth_name, "Degraded", timeout=60) test/e2e/tests/test_subscription.py:2077: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:952: in _wait_for_maas_auth_policy_phase cr = _get_cr("maasauthpolicy", name, namespace) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maasauthpolicy', name = 'e2e-status-degraded-auth' namespace = 'models-as-a-service' def _get_cr(kind, name, namespace=None): """Get a CR as dict, or None if not found. Retries on transient errors. Returns None only when the resource genuinely does not exist (server NotFound). Raises RuntimeError for other failures (RBAC, missing CRD, transport errors that persist after retries) so callers can distinguish infrastructure issues from true absence. """ namespace = namespace or _ns() max_retries = 3 retry_delay = 2 for attempt in range(max_retries): result = subprocess.run(["oc", "get", kind, name, "-n", namespace, "-o", "json"], capture_output=True, text=True) if result.returncode == 0: return json.loads(result.stdout) if attempt < max_retries - 1 and _is_transient_kubectl_error(result.stderr): log.warning( f"Transient kubectl error getting {kind}/{name} (attempt {attempt + 1}/{max_retries}): {result.stderr.strip()}" ) time.sleep(retry_delay * (attempt + 1)) continue # Terminal failure — distinguish not-found from other errors if _is_not_found_error(result.stderr): return None log.error( f"Failed to get {kind}/{name} in namespace '{namespace}' after {attempt + 1} attempts. " f"Last error: {result.stderr.strip()}" ) > raise RuntimeError( f"Failed to get {kind}/{name} in namespace '{namespace}': {result.stderr.strip()}" ) E RuntimeError: Failed to get maasauthpolicy/e2e-status-degraded-auth in namespace 'models-as-a-service': Unable to connect to the server: net/http: TLS handshake timeout test/e2e/tests/test_helper.py:390: RuntimeError During handling of the above exception, another exception occurred: self = <test_subscription.TestStatusReporting object at 0x7f276e591580> def test_authpolicy_degraded_status_with_partial_models(self): """ Test: MaaSAuthPolicy shows Degraded phase when some models are valid, some missing. Creates an auth policy with one valid and one missing model ref and verifies: - Phase is "Degraded" - authPolicies contains entry for the valid model (ready=true) """ ns = _ns() auth_name = "e2e-status-degraded-auth" sa_name = "e2e-status-degraded-auth-sa" missing_model = "nonexistent-model-auth-partial" try: _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) # Create auth policy with both valid and missing models _create_test_auth_policy(auth_name, [MODEL_REF, missing_model], users=[sa_user]) # Wait for auth policy to reach Degraded phase with polling cr = _wait_for_maas_auth_policy_phase(auth_name, "Degraded", timeout=60) status = cr.get("status", {}) auth_policies = status.get("authPolicies", []) log.info(f"AuthPolicy status: phase={status.get('phase')}, authPolicies={auth_policies}") # Should have at least one entry for the valid model if len(auth_policies) > 0: ready_count = sum(1 for ap in auth_policies if ap.get("ready") is True) log.info(f"Found {ready_count} ready authPolicies out of {len(auth_policies)}") log.info("✅ MaaSAuthPolicy Degraded status verified") finally: _delete_cr("maasauthpolicy", auth_name, namespace=ns) > _delete_sa(sa_name, namespace=MODEL_NAMESPACE) test/e2e/tests/test_subscription.py:2093: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:193: in _delete_sa result = subprocess.run( /usr/lib64/python3.9/subprocess.py:507: in run stdout, stderr = process.communicate(input, timeout=timeout) /usr/lib64/python3.9/subprocess.py:1134: in communicate stdout, stderr = self._communicate(input, endtime, timeout) /usr/lib64/python3.9/subprocess.py:1996: in _communicate self._check_timeout(endtime, orig_timeout, stdout, stderr) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <Popen: returncode: -9 args: ['oc', 'delete', 'sa', 'e2e-status-degraded-aut...> endtime = 50142.542046008, orig_timeout = 30, stdout_seq = [], stderr_seq = [] skip_check_and_raise = False def _check_timeout(self, endtime, orig_timeout, stdout_seq, stderr_seq, skip_check_and_raise=False): """Convenience for checking if a timeout has expired.""" if endtime is None: return if skip_check_and_raise or _time() > endtime: > raise TimeoutExpired( self.args, orig_timeout, output=b''.join(stdout_seq) if stdout_seq else None, stderr=b''.join(stderr_seq) if stderr_seq else None) E subprocess.TimeoutExpired: Command '['oc', 'delete', 'sa', 'e2e-status-degraded-auth-sa', '-n', 'llm', '--ignore-not-found']' timed out after 30 seconds /usr/lib64/python3.9/subprocess.py:1178: TimeoutExpiredself = <test_subscription.TestStatusReporting object at 0x7f276e5918e0> def test_subscription_status_transitions_on_model_deletion(self): """ Test: MaaSSubscription transitions from Active to Degraded/Failed when model is deleted. Creates a subscription with a temporary model, verifies Active status, then deletes the model and verifies status transitions appropriately. """ ns = _ns() subscription_name = "e2e-status-transition-sub" auth_name = "e2e-status-transition-auth" model_name = "e2e-temp-model-status" sa_name = "e2e-status-transition-sa" try: > _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) test/e2e/tests/test_subscription.py:2110: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ sa_name = 'e2e-status-transition-sa', namespace = 'llm', duration = '10m' def _create_sa_token(sa_name, namespace=None, duration="10m"): namespace = namespace or _ns() sa_result = subprocess.run( ["oc", "create", "sa", sa_name, "-n", namespace], capture_output=True, text=True ) if sa_result.returncode != 0 and "already exists" not in sa_result.stderr: > raise RuntimeError(f"Failed to create SA {sa_name}: {sa_result.stderr}") E RuntimeError: Failed to create SA e2e-status-transition-sa: error: failed to create serviceaccount: Post "https://ac9d8d277b501404e83ccf4bfd66001c-f4cdb3420478dd37.elb.us-east-1.amazonaws.com:6443/api/v1/namespaces/llm/serviceaccounts?fieldManager=kubectl-create&fieldValidation=Ignore": net/http: TLS handshake timeout test/e2e/tests/test_helper.py:179: RuntimeErrorself = <test_subscription.TestDegradedSubscriptionFiltering object at 0x7f276e0a5370> def test_degraded_healthy_model_allows_inference(self): """ Test: Inference to healthy model in Degraded subscription succeeds. Setup: 1. Create subscription with 1 valid + 1 missing model 2. Controller sets phase=Degraded, modelRefStatuses shows mixed health Verify: - Subscription is Degraded with one ready=true, one ready=false - Inference to the valid model succeeds (200) """ ns = _ns() subscription_name = "e2e-degraded-healthy-inf" auth_name = "e2e-degraded-healthy-inf-auth" sa_name = "e2e-degraded-healthy-inf-sa" missing_model = "nonexistent-model-inf" try: > oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) test/e2e/tests/test_subscription.py:2203: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ sa_name = 'e2e-degraded-healthy-inf-sa', namespace = 'llm', duration = '10m' def _create_sa_token(sa_name, namespace=None, duration="10m"): namespace = namespace or _ns() sa_result = subprocess.run( ["oc", "create", "sa", sa_name, "-n", namespace], capture_output=True, text=True ) if sa_result.returncode != 0 and "already exists" not in sa_result.stderr: > raise RuntimeError(f"Failed to create SA {sa_name}: {sa_result.stderr}") E RuntimeError: Failed to create SA e2e-degraded-healthy-inf-sa: error: failed to create serviceaccount: Post "https://ac9d8d277b501404e83ccf4bfd66001c-f4cdb3420478dd37.elb.us-east-1.amazonaws.com:6443/api/v1/namespaces/llm/serviceaccounts?fieldManager=kubectl-create&fieldValidation=Ignore": net/http: TLS handshake timeout test/e2e/tests/test_helper.py:179: RuntimeErrorself = <test_subscription.TestDegradedSubscriptionFiltering object at 0x7f276e0a56a0> def test_failed_subscription_blocks_inference(self): """ Test: Failed subscription blocks inference via OPA rule. Setup: 1. Create subscription with valid model (starts Active) 2. Create API key 3. Manually patch subscription to Failed phase 4. Verify inference is rejected by OPA (403) Note: We use manual patching because naturally creating a Failed subscription requires only invalid models, which don't have routes (404 before OPA runs). """ ns = _ns() subscription_name = "e2e-failed-sub-inf" auth_name = "e2e-failed-sub-inf-auth" sa_name = "e2e-failed-sub-inf-sa" try: > oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) test/e2e/tests/test_subscription.py:2282: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ sa_name = 'e2e-failed-sub-inf-sa', namespace = 'llm', duration = '10m' def _create_sa_token(sa_name, namespace=None, duration="10m"): namespace = namespace or _ns() sa_result = subprocess.run( ["oc", "create", "sa", sa_name, "-n", namespace], capture_output=True, text=True ) if sa_result.returncode != 0 and "already exists" not in sa_result.stderr: > raise RuntimeError(f"Failed to create SA {sa_name}: {sa_result.stderr}") E RuntimeError: Failed to create SA e2e-failed-sub-inf-sa: error: failed to create serviceaccount: Post "https://ac9d8d277b501404e83ccf4bfd66001c-f4cdb3420478dd37.elb.us-east-1.amazonaws.com:6443/api/v1/namespaces/llm/serviceaccounts?fieldManager=kubectl-create&fieldValidation=Ignore": net/http: TLS handshake timeout test/e2e/tests/test_helper.py:179: RuntimeErrorself = <test_subscription.TestDegradedSubscriptionFiltering object at 0x7f276e0a5f40> def test_models_endpoint_with_degraded_subscription_api_key(self): """ Test: /v1/models with API key bound to Degraded subscription. Verify behavior when querying models list with a Degraded subscription. Current implementation may succeed (showing valid models) or fail depending on selector implementation. """ ns = _ns() subscription_name = "e2e-degraded-models-apikey" auth_name = "e2e-degraded-models-apikey-auth" sa_name = "e2e-degraded-models-apikey-sa" missing_model = "nonexistent-model-apikey" try: > oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) test/e2e/tests/test_subscription.py:2393: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ sa_name = 'e2e-degraded-models-apikey-sa', namespace = 'llm', duration = '10m' def _create_sa_token(sa_name, namespace=None, duration="10m"): namespace = namespace or _ns() sa_result = subprocess.run( ["oc", "create", "sa", sa_name, "-n", namespace], capture_output=True, text=True ) if sa_result.returncode != 0 and "already exists" not in sa_result.stderr: > raise RuntimeError(f"Failed to create SA {sa_name}: {sa_result.stderr}") E RuntimeError: Failed to create SA e2e-degraded-models-apikey-sa: error: failed to create serviceaccount: Post "https://ac9d8d277b501404e83ccf4bfd66001c-f4cdb3420478dd37.elb.us-east-1.amazonaws.com:6443/api/v1/namespaces/llm/serviceaccounts?fieldManager=kubectl-create&fieldValidation=Ignore": net/http: TLS handshake timeout test/e2e/tests/test_helper.py:179: RuntimeError During handling of the above exception, another exception occurred: self = <test_subscription.TestDegradedSubscriptionFiltering object at 0x7f276e0a5f40> def test_models_endpoint_with_degraded_subscription_api_key(self): """ Test: /v1/models with API key bound to Degraded subscription. Verify behavior when querying models list with a Degraded subscription. Current implementation may succeed (showing valid models) or fail depending on selector implementation. """ ns = _ns() subscription_name = "e2e-degraded-models-apikey" auth_name = "e2e-degraded-models-apikey-auth" sa_name = "e2e-degraded-models-apikey-sa" missing_model = "nonexistent-model-apikey" try: oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) # Create auth policy _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) # Create subscription _create_test_subscription( subscription_name, [MODEL_REF, missing_model], users=[sa_user] ) _wait_reconcile(seconds=10) # Verify Degraded cr = _get_cr("maassubscription", subscription_name, namespace=ns) phase = cr.get("status", {}).get("phase") assert phase == "Degraded", f"Expected Degraded, got {phase}" # Create API key # oc_token already set from _create_sa_token above api_key = _create_api_key( oc_token, name="degraded-models", subscription=subscription_name ) # Call /v1/models url = f"{_maas_api_url()}/v1/models" headers = { "Authorization": f"Bearer {api_key}", "Content-Type": "application/json" } log.info(f"GET {url} with API key") r = _request_with_gateway_retry(requests.get, url, headers=headers) log.info(f"Response: {r.status_code}") # Should succeed - API key can list models from Degraded subscription assert r.status_code == 200, \ f"Expected 200 for /v1/models with Degraded subscription API key, got {r.status_code}: {r.text[:500]}" data = r.json() models = data.get("data", []) log.info(f"✅ /v1/models succeeded, returned {len(models)} models") # Gateway-only behavior may filter degraded subscriptions from /v1/models. # Validate success response shape rather than requiring model visibility. assert isinstance(models, list), "Expected /v1/models data to be a list" finally: _delete_cr("maassubscription", subscription_name, namespace=ns) _delete_cr("maasauthpolicy", auth_name, namespace=ns) > _delete_sa(sa_name, namespace=MODEL_NAMESPACE) test/e2e/tests/test_subscription.py:2448: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:193: in _delete_sa result = subprocess.run( /usr/lib64/python3.9/subprocess.py:507: in run stdout, stderr = process.communicate(input, timeout=timeout) /usr/lib64/python3.9/subprocess.py:1134: in communicate stdout, stderr = self._communicate(input, endtime, timeout) /usr/lib64/python3.9/subprocess.py:1996: in _communicate self._check_timeout(endtime, orig_timeout, stdout, stderr) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <Popen: returncode: -9 args: ['oc', 'delete', 'sa', 'e2e-degraded-models-api...> endtime = 50380.587204748, orig_timeout = 30, stdout_seq = [], stderr_seq = [] skip_check_and_raise = False def _check_timeout(self, endtime, orig_timeout, stdout_seq, stderr_seq, skip_check_and_raise=False): """Convenience for checking if a timeout has expired.""" if endtime is None: return if skip_check_and_raise or _time() > endtime: > raise TimeoutExpired( self.args, orig_timeout, output=b''.join(stdout_seq) if stdout_seq else None, stderr=b''.join(stderr_seq) if stderr_seq else None) E subprocess.TimeoutExpired: Command '['oc', 'delete', 'sa', 'e2e-degraded-models-apikey-sa', '-n', 'llm', '--ignore-not-found']' timed out after 30 seconds /usr/lib64/python3.9/subprocess.py:1178: TimeoutExpired