self = <test_api_keys.TestAPIKeySubscriptionPhases object at 0x7f455a6ee670> def test_reject_key_for_unreconciled_subscription(self): """ API key creation is rejected for unreconciled subscription (empty phase). Note: Temporarily sets webhook failurePolicy to Ignore to allow creating resources while controller is down, then restores to Fail. """ ns = _ns() subscription_name = "e2e-apikey-unreconciled-sub" auth_name = "e2e-apikey-unreconciled-auth" sa_name = "e2e-apikey-unreconciled-sa" webhook_name = "maas-validating-webhook-configuration" try: # Create service account and get token oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) # Temporarily set webhook failurePolicy to Ignore # This allows creates to succeed when controller/webhook is unavailable # Find webhook indices dynamically by name to avoid brittleness result = subprocess.run( ["oc", "get", "validatingwebhookconfiguration", webhook_name, "-o", "json"], capture_output=True, text=True, check=True ) webhook_config = json.loads(result.stdout) patch_ops = [] for idx, webhook in enumerate(webhook_config.get("webhooks", [])): if webhook.get("name") in ["vmaassubscription.kb.io", "vmaasauthpolicy.kb.io"]: patch_ops.append({"op": "replace", "path": f"/webhooks/{idx}/failurePolicy", "value": "Ignore"}) subprocess.run( ["oc", "patch", "validatingwebhookconfiguration", webhook_name, "--type=json", "-p", json.dumps(patch_ops)], capture_output=True, text=True, check=True ) # Scale down controller to prevent reconciliation > _scale_controller_down() test/e2e/tests/test_api_keys.py:1393: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:987: in _scale_controller_down _scale_controller(0, namespace, timeout) test/e2e/tests/test_helper.py:944: in _scale_controller result = subprocess.run( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ input = None, capture_output = True, timeout = 60, check = True popenargs = (['oc', 'scale', 'deployment', 'maas-controller', '--replicas=0', '-n', ...],) kwargs = {'stderr': -1, 'stdout': -1, 'text': True} process = <Popen: returncode: 1 args: ['oc', 'scale', 'deployment', 'maas-controller',...> stdout = '' stderr = 'Error from server (Timeout): Timeout: request did not complete within requested timeout - context deadline exceeded\n' retcode = 1 def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: > raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) E subprocess.CalledProcessError: Command '['oc', 'scale', 'deployment', 'maas-controller', '--replicas=0', '-n', 'opendatahub']' returned non-zero exit status 1. /usr/lib64/python3.9/subprocess.py:528: CalledProcessError During handling of the above exception, another exception occurred: self = <test_api_keys.TestAPIKeySubscriptionPhases object at 0x7f455a6ee670> def test_reject_key_for_unreconciled_subscription(self): """ API key creation is rejected for unreconciled subscription (empty phase). Note: Temporarily sets webhook failurePolicy to Ignore to allow creating resources while controller is down, then restores to Fail. """ ns = _ns() subscription_name = "e2e-apikey-unreconciled-sub" auth_name = "e2e-apikey-unreconciled-auth" sa_name = "e2e-apikey-unreconciled-sa" webhook_name = "maas-validating-webhook-configuration" try: # Create service account and get token oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) # Temporarily set webhook failurePolicy to Ignore # This allows creates to succeed when controller/webhook is unavailable # Find webhook indices dynamically by name to avoid brittleness result = subprocess.run( ["oc", "get", "validatingwebhookconfiguration", webhook_name, "-o", "json"], capture_output=True, text=True, check=True ) webhook_config = json.loads(result.stdout) patch_ops = [] for idx, webhook in enumerate(webhook_config.get("webhooks", [])): if webhook.get("name") in ["vmaassubscription.kb.io", "vmaasauthpolicy.kb.io"]: patch_ops.append({"op": "replace", "path": f"/webhooks/{idx}/failurePolicy", "value": "Ignore"}) subprocess.run( ["oc", "patch", "validatingwebhookconfiguration", webhook_name, "--type=json", "-p", json.dumps(patch_ops)], capture_output=True, text=True, check=True ) # Scale down controller to prevent reconciliation _scale_controller_down() # Create resources (webhook unavailable but Ignore policy allows creates) _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) # Verify subscription is unreconciled (empty phase) cr = _get_cr("maassubscription", subscription_name, namespace=ns) phase = cr.get("status", {}).get("phase", "") assert phase == "", f"Expected empty phase, got: {phase}" log.info("✅ Subscription is unreconciled (empty phase)") # Try to create API key (should fail with 400) response = requests.post( f"{_maas_api_url()}/v1/api-keys", headers={ "Authorization": f"Bearer {oc_token}", "Content-Type": "application/json" }, json={ "name": "unreconciled-sub-test", "subscription": subscription_name }, timeout=TIMEOUT, verify=TLS_VERIFY, ) assert response.status_code == 400, \ f"Expected 400 for unreconciled subscription, got {response.status_code}: {response.text}" response_data = response.json() assert "code" in response_data and response_data["code"] == "subscription_not_ready", \ f"Expected subscription_not_ready error code, got: {response_data}" log.info("✅ API key creation rejected for unreconciled subscription") finally: # Restore webhook failurePolicy to Fail # Wrap in try/except to guarantee controller scale-up and cleanup happen webhook_restore_error = None try: result = subprocess.run( ["oc", "get", "validatingwebhookconfiguration", webhook_name, "-o", "json"], capture_output=True, text=True ) if result.returncode != 0: raise RuntimeError( f"Failed to get webhook configuration {webhook_name} for restoration: {result.stderr}" ) webhook_config = json.loads(result.stdout) patch_ops = [] for idx, webhook in enumerate(webhook_config.get("webhooks", [])): if webhook.get("name") in ["vmaassubscription.kb.io", "vmaasauthpolicy.kb.io"]: patch_ops.append({"op": "replace", "path": f"/webhooks/{idx}/failurePolicy", "value": "Fail"}) if not patch_ops: raise RuntimeError( f"No matching webhooks found to restore in {webhook_name}. " f"Expected vmaassubscription.kb.io and vmaasauthpolicy.kb.io. " f"Found: {[w.get('name') for w in webhook_config.get('webhooks', [])]}" ) restore_result = subprocess.run( ["oc", "patch", "validatingwebhookconfiguration", webhook_name, "--type=json", "-p", json.dumps(patch_ops)], capture_output=True, text=True ) if restore_result.returncode != 0: raise RuntimeError( f"Webhook left in Ignore state! Failed to restore webhook {webhook_name} " f"with patch_ops {patch_ops}: {restore_result.stderr}" ) except Exception as e: webhook_restore_error = e log.error(f"Exception during webhook restoration: {e}") # Always scale controller back up and cleanup, even if webhook restore failed _scale_controller_up() _delete_cr("maassubscription", subscription_name, namespace=ns) _delete_cr("maasauthpolicy", auth_name, namespace=ns) _delete_sa(sa_name, namespace=MODEL_NAMESPACE) _wait_reconcile() # Only raise webhook restore error after cleanup is complete if webhook_restore_error: > raise webhook_restore_error test/e2e/tests/test_api_keys.py:1478: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <test_api_keys.TestAPIKeySubscriptionPhases object at 0x7f455a6ee670> def test_reject_key_for_unreconciled_subscription(self): """ API key creation is rejected for unreconciled subscription (empty phase). Note: Temporarily sets webhook failurePolicy to Ignore to allow creating resources while controller is down, then restores to Fail. """ ns = _ns() subscription_name = "e2e-apikey-unreconciled-sub" auth_name = "e2e-apikey-unreconciled-auth" sa_name = "e2e-apikey-unreconciled-sa" webhook_name = "maas-validating-webhook-configuration" try: # Create service account and get token oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) # Temporarily set webhook failurePolicy to Ignore # This allows creates to succeed when controller/webhook is unavailable # Find webhook indices dynamically by name to avoid brittleness result = subprocess.run( ["oc", "get", "validatingwebhookconfiguration", webhook_name, "-o", "json"], capture_output=True, text=True, check=True ) webhook_config = json.loads(result.stdout) patch_ops = [] for idx, webhook in enumerate(webhook_config.get("webhooks", [])): if webhook.get("name") in ["vmaassubscription.kb.io", "vmaasauthpolicy.kb.io"]: patch_ops.append({"op": "replace", "path": f"/webhooks/{idx}/failurePolicy", "value": "Ignore"}) subprocess.run( ["oc", "patch", "validatingwebhookconfiguration", webhook_name, "--type=json", "-p", json.dumps(patch_ops)], capture_output=True, text=True, check=True ) # Scale down controller to prevent reconciliation _scale_controller_down() # Create resources (webhook unavailable but Ignore policy allows creates) _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) # Verify subscription is unreconciled (empty phase) cr = _get_cr("maassubscription", subscription_name, namespace=ns) phase = cr.get("status", {}).get("phase", "") assert phase == "", f"Expected empty phase, got: {phase}" log.info("✅ Subscription is unreconciled (empty phase)") # Try to create API key (should fail with 400) response = requests.post( f"{_maas_api_url()}/v1/api-keys", headers={ "Authorization": f"Bearer {oc_token}", "Content-Type": "application/json" }, json={ "name": "unreconciled-sub-test", "subscription": subscription_name }, timeout=TIMEOUT, verify=TLS_VERIFY, ) assert response.status_code == 400, \ f"Expected 400 for unreconciled subscription, got {response.status_code}: {response.text}" response_data = response.json() assert "code" in response_data and response_data["code"] == "subscription_not_ready", \ f"Expected subscription_not_ready error code, got: {response_data}" log.info("✅ API key creation rejected for unreconciled subscription") finally: # Restore webhook failurePolicy to Fail # Wrap in try/except to guarantee controller scale-up and cleanup happen webhook_restore_error = None try: result = subprocess.run( ["oc", "get", "validatingwebhookconfiguration", webhook_name, "-o", "json"], capture_output=True, text=True ) if result.returncode != 0: > raise RuntimeError( f"Failed to get webhook configuration {webhook_name} for restoration: {result.stderr}" ) E RuntimeError: Failed to get webhook configuration maas-validating-webhook-configuration for restoration: Unable to connect to the server: net/http: TLS handshake timeout test/e2e/tests/test_api_keys.py:1437: RuntimeError/workspace/source/test/e2e/tests/test_namespace_scoping.py:212: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:245: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:283: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:320: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:377: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_namespace_scoping.py:453: test_namespace_scoping validates single-tenant dormant mode; skipped when ENABLE_TENANT_NAMESPACE_DISCOVERY=true/workspace/source/test/e2e/tests/test_subscription.py:1056: gateway-only mode: per-model AuthPolicy is not created/workspace/source/test/e2e/tests/test_tenant.py:127: Tenant not Active (e.g. Degraded); payload-processing not asserted/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:186: OIDC_ISSUER_URL not set; per-tenant OIDC E2E requires external OIDC deploy/workspace/source/test/e2e/tests/test_tenant_namespace_discovery.py:358: Dormant-mode test mutates controller flags; set ENABLE_TENANT_DISCOVERY_DORMANT_E2E=true/workspace/source/test/e2e/tests/test_gateway_scoped_authpolicy.py:170: gateway-default-auth not present (maas-gateway-auth is active); scoping is validated by unit tests/workspace/source/test/e2e/tests/test_tenant_auth_isolation.py:158: OIDC_TOKEN_TENANT_A and OIDC_TOKEN_TENANT_B are required for per-tenant OIDC validation/workspace/source/test/e2e/tests/test_external_oidc.py:238: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:249: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:263: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:276: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:289: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:321: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:365: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:381: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:391: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:414: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:422: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:427: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:440: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:487: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:523: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:588: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:619: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:645: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:689: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:713: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:766: EXTERNAL_OIDC is not true/workspace/source/test/e2e/tests/test_external_oidc.py:814: EXTERNAL_OIDC is not true