/workspace/source/test/e2e/tests/test_crd_watch_resilience.py:224: requires a disposable cluster: deletes the cluster-scoped KServe CRD; set E2E_RUN_CRD_WATCH_RESILIENCE=true to opt in/workspace/source/test/e2e/tests/test_crd_watch_resilience.py:262: requires a disposable cluster: deletes the cluster-scoped KServe CRD; set E2E_RUN_CRD_WATCH_RESILIENCE=true to opt in/workspace/source/test/e2e/tests/test_crd_watch_resilience.py:312: requires a disposable cluster: deletes the cluster-scoped KServe CRD; set E2E_RUN_CRD_WATCH_RESILIENCE=true to opt in/workspace/source/test/e2e/tests/test_subscription.py:961: no Kuadrant WasmPlugin for gateway maas-default-gateway/workspace/source/test/e2e/tests/test_subscription.py:1748: gateway-only mode: per-model AuthPolicy is not createdself = <test_subscription.TestStatusReporting object at 0x7fecd6e435d0> @pytest.mark.serial def test_subscription_degraded_trlp_blocks_inference(self): """ Test: Degraded subscription with TRLP not ready blocks inference. This test verifies that when a subscription enters Degraded phase due to TokenRateLimitPolicy not being ready (e.g., Kuadrant controller down), inference requests are blocked with appropriate error to prevent rate limits from being bypassed. Uses pre-deployed e2e-trlp-test-simulated model to avoid TRLP sharing with concurrent tests. Test flow: 1. Create auth policy with Kuadrant up — wait for Active + Enforced 2. Scale down Kuadrant controller 3. Create subscription — TRLP created but not accepted 4. Wait for subscription to enter Degraded phase (TRLP ready=false) 5. Create API key and verify inference is blocked (403 Forbidden) 6. Scale Kuadrant controller back up 7. Wait for subscription to reach Active phase (TRLP ready=true) 8. Verify inference works (200 OK) """ ns = _ns() subscription_name = "e2e-trlp-degraded-sub" auth_name = "e2e-trlp-degraded-auth" sa_name = "e2e-trlp-degraded-sa" try: # Step 1: Create auth policy with Kuadrant UP so it can reach Active + Enforced. # The controller gates MaaSAuthPolicy on gateway AuthPolicy enforcement # (RHOAIENG-79568), so Kuadrant must be running for the phase to reach Active. log.info("Step 1: Creating auth policy with Kuadrant up...") sa_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) _create_test_auth_policy(auth_name, TRLP_TEST_MODEL_REF, users=[sa_user]) _wait_for_maas_auth_policy_phase( auth_name, "Active", require_auth_policies=False, ) log.info("Step 1: Auth policy Active + Enforced") # Step 2: Scale down Kuadrant controller BEFORE creating subscription. # The auth policy is already Active and the gateway AuthPolicy is Enforced. # Creating a subscription does NOT change the gateway AuthPolicy spec # (only MaaSAuthPolicy subjects feed the allowlist), so no enforcement # re-check is triggered. log.info("Step 2: Scaling down Kuadrant controller...") _scale_kuadrant_controller_down() # Step 3: Create subscription — TRLP cannot be reconciled with Kuadrant down. log.info("Step 3: Creating subscription with Kuadrant controller down...") _create_test_subscription(subscription_name, TRLP_TEST_MODEL_REF, users=[sa_user]) # Step 4: Wait for subscription to reach Degraded phase with TRLP not ready log.info("Step 4: Waiting for subscription to enter Degraded phase (TRLP not ready)...") cr = _wait_for_maas_subscription_phase(subscription_name, "Degraded") _wait_for_subscription_trlp_status(subscription_name, expected_ready=False) status = cr.get("status", {}) trlp_statuses = status.get("tokenRateLimitStatuses", []) log.info(f"Subscription Degraded: phase={status.get('phase')}, trlpStatuses={trlp_statuses}") # Verify at least one TRLP is not ready assert len(trlp_statuses) > 0, "Expected at least one TRLP status" assert any(not trlp.get("ready") for trlp in trlp_statuses), "Expected at least one TRLP to be not ready" log.info("✅ Subscription in Degraded phase with TRLP not ready") # Step 5: Create API key and verify inference is blocked log.info("Step 5: Creating API key and verifying inference is blocked...") api_key = _create_api_key(sa_token, name="e2e-trlp-test-key", subscription=subscription_name) resp = _poll_status(api_key, 403, path=TRLP_TEST_MODEL_PATH, model_name=TRLP_TEST_MODEL_ID, timeout=60) assert resp.status_code == 403, f"Expected 403 Forbidden for Degraded subscription with TRLP not ready, got {resp.status_code}: {resp.text}" log.info("✅ Inference blocked for Degraded subscription with TRLP not ready") # Step 6: Scale Kuadrant controller back up log.info("Step 6: Scaling Kuadrant controller back up...") > _scale_kuadrant_controller_up() test/e2e/tests/test_subscription.py:2815: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:2324: in _scale_kuadrant_controller_up _scale_kuadrant_controller(1, namespace, timeout) test/e2e/tests/test_helper.py:2303: in _scale_kuadrant_controller subprocess.run( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ input = None, capture_output = True, timeout = None, check = True popenargs = (['oc', 'wait', '--for=condition=ready', 'pod', '-l', 'control-plane=controller-manager', ...],) kwargs = {'text': True, 'stdout': -1, 'stderr': -1} def run(*popenargs, input=None, capture_output=False, timeout=None, check=False, **kwargs): """Run command with arguments and return a CompletedProcess instance. The returned instance will have attributes args, returncode, stdout and stderr. By default, stdout and stderr are not captured, and those attributes will be None. Pass stdout=PIPE and/or stderr=PIPE in order to capture them, or pass capture_output=True to capture both. If check is True and the exit code was non-zero, it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute, and output & stderr attributes if those streams were captured. If timeout is given, and the process takes too long, a TimeoutExpired exception will be raised. There is an optional argument "input", allowing you to pass bytes or a string to the subprocess's stdin. If you use this argument you may not also use the Popen constructor's "stdin" argument, as it will be used internally. By default, all communication is in bytes, and therefore any "input" should be bytes, and the stdout and stderr will be bytes. If in text mode, any "input" should be a string, and stdout and stderr will be strings decoded according to locale encoding, or by "encoding" if set. Text mode is triggered by setting any of text, encoding, errors or universal_newlines. The other arguments are the same as for the Popen constructor. """ if input is not None: if kwargs.get('stdin') is not None: raise ValueError('stdin and input arguments may not both be used.') kwargs['stdin'] = PIPE if capture_output: if kwargs.get('stdout') is not None or kwargs.get('stderr') is not None: raise ValueError('stdout and stderr arguments may not be used ' 'with capture_output.') kwargs['stdout'] = PIPE kwargs['stderr'] = PIPE with Popen(*popenargs, **kwargs) as process: try: stdout, stderr = process.communicate(input, timeout=timeout) except TimeoutExpired as exc: process.kill() if _mswindows: # Windows accumulates the output in a single blocking # read() call run on child threads, with the timeout # being done in a join() on those threads. communicate() # _after_ kill() is required to collect that and add it # to the exception. exc.stdout, exc.stderr = process.communicate() else: # POSIX _communicate already populated the output so # far into the TimeoutExpired exception. process.wait() raise except: # Including KeyboardInterrupt, communicate handled that. process.kill() # We don't call process.wait() as .__exit__ does that for us. raise retcode = process.poll() if check and retcode: > raise CalledProcessError(retcode, process.args, output=stdout, stderr=stderr) E subprocess.CalledProcessError: Command '['oc', 'wait', '--for=condition=ready', 'pod', '-l', 'control-plane=controller-manager', '-n', 'kuadrant-system', '--timeout=60s']' returned non-zero exit status 1. /usr/lib64/python3.11/subprocess.py:571: CalledProcessError During handling of the above exception, another exception occurred: self = <test_subscription.TestStatusReporting object at 0x7fecd6e435d0> @pytest.mark.serial def test_subscription_degraded_trlp_blocks_inference(self): """ Test: Degraded subscription with TRLP not ready blocks inference. This test verifies that when a subscription enters Degraded phase due to TokenRateLimitPolicy not being ready (e.g., Kuadrant controller down), inference requests are blocked with appropriate error to prevent rate limits from being bypassed. Uses pre-deployed e2e-trlp-test-simulated model to avoid TRLP sharing with concurrent tests. Test flow: 1. Create auth policy with Kuadrant up — wait for Active + Enforced 2. Scale down Kuadrant controller 3. Create subscription — TRLP created but not accepted 4. Wait for subscription to enter Degraded phase (TRLP ready=false) 5. Create API key and verify inference is blocked (403 Forbidden) 6. Scale Kuadrant controller back up 7. Wait for subscription to reach Active phase (TRLP ready=true) 8. Verify inference works (200 OK) """ ns = _ns() subscription_name = "e2e-trlp-degraded-sub" auth_name = "e2e-trlp-degraded-auth" sa_name = "e2e-trlp-degraded-sa" try: # Step 1: Create auth policy with Kuadrant UP so it can reach Active + Enforced. # The controller gates MaaSAuthPolicy on gateway AuthPolicy enforcement # (RHOAIENG-79568), so Kuadrant must be running for the phase to reach Active. log.info("Step 1: Creating auth policy with Kuadrant up...") sa_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) _create_test_auth_policy(auth_name, TRLP_TEST_MODEL_REF, users=[sa_user]) _wait_for_maas_auth_policy_phase( auth_name, "Active", require_auth_policies=False, ) log.info("Step 1: Auth policy Active + Enforced") # Step 2: Scale down Kuadrant controller BEFORE creating subscription. # The auth policy is already Active and the gateway AuthPolicy is Enforced. # Creating a subscription does NOT change the gateway AuthPolicy spec # (only MaaSAuthPolicy subjects feed the allowlist), so no enforcement # re-check is triggered. log.info("Step 2: Scaling down Kuadrant controller...") _scale_kuadrant_controller_down() # Step 3: Create subscription — TRLP cannot be reconciled with Kuadrant down. log.info("Step 3: Creating subscription with Kuadrant controller down...") _create_test_subscription(subscription_name, TRLP_TEST_MODEL_REF, users=[sa_user]) # Step 4: Wait for subscription to reach Degraded phase with TRLP not ready log.info("Step 4: Waiting for subscription to enter Degraded phase (TRLP not ready)...") cr = _wait_for_maas_subscription_phase(subscription_name, "Degraded") _wait_for_subscription_trlp_status(subscription_name, expected_ready=False) status = cr.get("status", {}) trlp_statuses = status.get("tokenRateLimitStatuses", []) log.info(f"Subscription Degraded: phase={status.get('phase')}, trlpStatuses={trlp_statuses}") # Verify at least one TRLP is not ready assert len(trlp_statuses) > 0, "Expected at least one TRLP status" assert any(not trlp.get("ready") for trlp in trlp_statuses), "Expected at least one TRLP to be not ready" log.info("✅ Subscription in Degraded phase with TRLP not ready") # Step 5: Create API key and verify inference is blocked log.info("Step 5: Creating API key and verifying inference is blocked...") api_key = _create_api_key(sa_token, name="e2e-trlp-test-key", subscription=subscription_name) resp = _poll_status(api_key, 403, path=TRLP_TEST_MODEL_PATH, model_name=TRLP_TEST_MODEL_ID, timeout=60) assert resp.status_code == 403, f"Expected 403 Forbidden for Degraded subscription with TRLP not ready, got {resp.status_code}: {resp.text}" log.info("✅ Inference blocked for Degraded subscription with TRLP not ready") # Step 6: Scale Kuadrant controller back up log.info("Step 6: Scaling Kuadrant controller back up...") _scale_kuadrant_controller_up() # Step 7: Wait for subscription to reach Active phase with TRLP ready log.info("Step 7: Waiting for subscription to reach Active phase (TRLP ready)...") _wait_for_maas_subscription_phase(subscription_name, "Active") _wait_for_subscription_trlp_status(subscription_name, expected_ready=True) # Drain AuthConfig backlog from the Kuadrant-down window before HTTP checks. _wait_for_gateway_auth_enforced() cr = _get_cr("maassubscription", subscription_name, namespace=ns) status = cr.get("status", {}) trlp_statuses = status.get("tokenRateLimitStatuses", []) log.info(f"Subscription Active: phase={status.get('phase')}, trlpStatuses={trlp_statuses}") # Verify all TRLPs are now ready assert all(trlp.get("ready") for trlp in trlp_statuses), "Expected all TRLPs to be ready" log.info("✅ Subscription returned to Active phase with all TRLPs ready") # Step 8: Verify inference works (poll to allow Envoy config propagation) log.info("Step 8: Verifying inference works with Active subscription...") resp = _poll_status(api_key, 200, path=TRLP_TEST_MODEL_PATH, model_name=TRLP_TEST_MODEL_ID, timeout=60) assert resp.status_code == 200, f"Expected 200 OK for Active subscription, got {resp.status_code}: {resp.text}" log.info("✅ Inference works with Active subscription after Kuadrant recovery") log.info("✅ TRLP validation e2e test complete") finally: # Ensure Kuadrant controller is scaled back up even if test fails try: log.info("Cleanup: Ensuring Kuadrant controller is scaled up...") _scale_kuadrant_controller_up() except Exception as e: log.warning(f"Failed to scale Kuadrant controller up during cleanup: {e}") # Revoke API key try: oc_token = _get_cluster_token() _revoke_api_key(oc_token, "e2e-trlp-test-key") except Exception as e: log.warning(f"Failed to revoke API key during cleanup: {e}") # Clean up resources (but not the model - it's pre-deployed) _delete_cr("maassubscription", subscription_name, namespace=ns) _delete_cr("maasauthpolicy", auth_name, namespace=ns) _delete_sa(sa_name, namespace=MODEL_NAMESPACE) > _wait_for_cr_absent("maassubscription", subscription_name) test/e2e/tests/test_subscription.py:2860: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maassubscription', name = 'e2e-trlp-degraded-sub' namespace = 'models-as-a-service', timeout = 30, poll_interval = 2 def _wait_for_cr_absent(kind, name, namespace=None, timeout=30, poll_interval=2): """Wait until a CR is deleted (no longer found by the API server).""" namespace = namespace or _ns() deadline = time.time() + timeout while time.time() < deadline: if _get_cr(kind, name, namespace) is None: return time.sleep(poll_interval) > raise TimeoutError( f"{kind}/{name} in {namespace} still exists after {timeout}s" ) E TimeoutError: maassubscription/e2e-trlp-degraded-sub in models-as-a-service still exists after 30s test/e2e/tests/test_helper.py:2161: TimeoutErrorself = <test_subscription.TestDegradedSubscriptionFiltering object at 0x7fecd6e3b190> @pytest.mark.serial def test_failed_subscription_blocks_inference(self): """ Test: Failed subscription blocks inference via OPA rule. Setup: 1. Create subscription with valid model (starts Active) 2. Create API key 3. Manually patch subscription to Failed phase 4. Verify inference is rejected by OPA (403) Note: We use manual patching because naturally creating a Failed subscription requires only invalid models, which don't have routes (404 before OPA runs). """ ns = _ns() subscription_name = "e2e-failed-sub-inf" auth_name = "e2e-failed-sub-inf-auth" sa_name = "e2e-failed-sub-inf-sa" try: oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) # Create auth policy for valid model > _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) test/e2e/tests/test_subscription.py:3095: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:1229: in _create_test_auth_policy _apply_cr({ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ cr_dict = {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSAuthPolicy', 'metadata': {'name': 'e2e-failed-sub-inf-auth...ated', 'namespace': 'llm'}], 'subjects': {'users': ['system:serviceaccount:llm:e2e-failed-sub-inf-sa'], 'groups': []}}} def _apply_cr(cr_dict): result = _run_oc(["oc", "apply", "-f", "-"], input_text=json.dumps(cr_dict)) if result.returncode != 0: meta = cr_dict.get("metadata", {}) > raise RuntimeError( f"`oc apply` failed for {cr_dict.get('kind')} " f"{meta.get('namespace', '<cluster>')}/{meta.get('name')}: " f"{result.stderr.strip() or result.stdout.strip()}" ) E RuntimeError: `oc apply` failed for MaaSAuthPolicy models-as-a-service/e2e-failed-sub-inf-auth: error: error when retrieving current configuration of: E Resource: "maas.opendatahub.io/v1alpha1, Resource=maasauthpolicies", GroupVersionKind: "maas.opendatahub.io/v1alpha1, Kind=MaaSAuthPolicy" E Name: "e2e-failed-sub-inf-auth", Namespace: "models-as-a-service" E from server for: "STDIN": Get "https://a1ae40bdd1dc145c69148e4a7dfcf45a-951b2e8c75f8896d.elb.us-east-2.amazonaws.com:6443/apis/maas.opendatahub.io/v1alpha1/namespaces/models-as-a-service/maasauthpolicies/e2e-failed-sub-inf-auth": dial tcp: lookup a1ae40bdd1dc145c69148e4a7dfcf45a-951b2e8c75f8896d.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host test/e2e/tests/test_helper.py:894: RuntimeError During handling of the above exception, another exception occurred: self = <test_subscription.TestDegradedSubscriptionFiltering object at 0x7fecd6e3b190> @pytest.mark.serial def test_failed_subscription_blocks_inference(self): """ Test: Failed subscription blocks inference via OPA rule. Setup: 1. Create subscription with valid model (starts Active) 2. Create API key 3. Manually patch subscription to Failed phase 4. Verify inference is rejected by OPA (403) Note: We use manual patching because naturally creating a Failed subscription requires only invalid models, which don't have routes (404 before OPA runs). """ ns = _ns() subscription_name = "e2e-failed-sub-inf" auth_name = "e2e-failed-sub-inf-auth" sa_name = "e2e-failed-sub-inf-sa" try: oc_token = _create_sa_token(sa_name, namespace=MODEL_NAMESPACE) sa_user = _sa_to_user(sa_name, namespace=MODEL_NAMESPACE) # Create auth policy for valid model _create_test_auth_policy(auth_name, MODEL_REF, users=[sa_user]) # Create subscription with valid model (will be Active) _create_test_subscription(subscription_name, MODEL_REF, users=[sa_user]) cr = _wait_for_maas_subscription_phase(subscription_name, "Active") # This test overrides controller-owned status to exercise the gateway # rejection path. Let initial TRLP/discovery reconciliation settle first, # and run serially so a queued reconcile from parallel test activity does # not overwrite the injected Failed phase before the inference request. _wait_for_subscription_inference_ready( subscription_name, MODEL_REF, namespace=ns, model_namespace=MODEL_NAMESPACE, ) # Verify it starts as Active phase = cr.get("status", {}).get("phase") log.info(f"Initial phase: {phase}") assert phase == "Active", f"Expected Active initially, got {phase}" # Create API key while Active api_key = _create_api_key( oc_token, name="failed-sub-test", subscription=subscription_name ) # Verify inference works while Active log.info("Testing inference while Active...") r = _inference(api_key, path=MODEL_PATH, model_name=MODEL_NAME) assert r.status_code == 200, f"Expected 200 while Active, got {r.status_code}: {r.text[:200]}" log.info("✅ Inference works with Active subscription") # Manually patch subscription to Failed phase import subprocess import json from datetime import datetime log.info("Manually patching subscription to Failed phase...") patch_data = { "status": { "phase": "Failed", "conditions": [ { "type": "Ready", "status": "False", "reason": "Failed", "message": "Subscription failed", "lastTransitionTime": datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ") } ], "modelRefStatuses": [ { "name": MODEL_REF, "namespace": MODEL_NAMESPACE, "ready": False, "reason": "ReconcileFailed", "message": "Model failed" } ] } } cmd = [ "kubectl", "patch", "maassubscription", subscription_name, "-n", ns, "--type=merge", "--subresource=status", "-p", json.dumps(patch_data), "-o", "json", ] result = subprocess.run(cmd, capture_output=True, text=True) assert result.returncode == 0, f"Failed to patch to Failed phase: {result.stderr}" # Read the object returned by the patch itself; a second API request # leaves extra time for any queued controller reconciliation to win. patched_cr = json.loads(result.stdout) phase = patched_cr.get("status", {}).get("phase") assert phase == "Failed", f"Expected Failed phase after patch, got {phase}" log.info("✅ Subscription patched to Failed phase") # Test inference with Failed subscription - should be rejected by OPA log.info("Testing inference with Failed subscription...") r = _inference(api_key, path=MODEL_PATH, model_name=MODEL_NAME) log.info(f"Response: status={r.status_code}, body={r.text[:200]}") # Failed phase should be rejected by OPA rule (403 or error message) if r.status_code == 200: assert "denied" in r.text.lower() or "access" in r.text.lower(), f"Expected access denied message, got: {r.text[:200]}" else: assert r.status_code == 403, f"Expected 403 for Failed subscription, got {r.status_code}: {r.text[:200]}" log.info("✅ Inference with Failed subscription correctly rejected by OPA") finally: _delete_cr("maassubscription", subscription_name, namespace=ns) _delete_cr("maasauthpolicy", auth_name, namespace=ns) _delete_sa(sa_name, namespace=MODEL_NAMESPACE) > _wait_for_cr_absent("maassubscription", subscription_name) test/e2e/tests/test_subscription.py:3195: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_helper.py:2158: in _wait_for_cr_absent if _get_cr(kind, name, namespace) is None: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ kind = 'maassubscription', name = 'e2e-failed-sub-inf' namespace = 'models-as-a-service', oc_timeout = None, oc_retry_budget = None def _get_cr(kind, name, namespace=None, *, oc_timeout=None, oc_retry_budget=None): """Get a CR as dict, or None if not found. Retries on transient errors. Returns None only when the resource genuinely does not exist (server NotFound). Raises RuntimeError for other failures (RBAC, missing CRD, transport errors that persist after retries) so callers can distinguish infrastructure issues from true absence. """ namespace = namespace or _ns() result = _run_oc( ["oc", "get", kind, name, "-n", namespace, "-o", "json"], timeout=oc_timeout, retry_budget=oc_retry_budget, ) if result.returncode == 0: return json.loads(result.stdout) if _is_not_found_error(result.stderr): return None log.error(f"Failed to get {kind}/{name} in namespace '{namespace}': {result.stderr.strip()}") > raise RuntimeError( f"Failed to get {kind}/{name} in namespace '{namespace}': {result.stderr.strip()}" ) E RuntimeError: Failed to get maassubscription/e2e-failed-sub-inf in namespace 'models-as-a-service': Unable to connect to the server: dial tcp: lookup a1ae40bdd1dc145c69148e4a7dfcf45a-951b2e8c75f8896d.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host test/e2e/tests/test_helper.py:1001: RuntimeError@pytest.fixture(scope="module", autouse=True) def require_tenant_crd(): r = _oc_run(["get", "crd", TENANT_CRD]) if r.returncode != 0: if _oc_output_not_found(r): pytest.skip( f"Missing CRD {TENANT_CRD} (transitional skip: install maas-controller manifests " f"so CRDs exist; then controller creates {TENANT_NAME})." ) combined = (r.stderr or "") + (r.stdout or "") > pytest.fail(f"`oc get crd {TENANT_CRD}` failed: {combined.strip()}") E Failed: `oc get crd maastenantconfigs.maas.opendatahub.io` failed: Unable to connect to the server: dial tcp: lookup a1ae40bdd1dc145c69148e4a7dfcf45a-951b2e8c75f8896d.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host test/e2e/tests/test_tenant.py:95: Failed@pytest.fixture(scope="module", autouse=True) def require_tenant_crd(): r = _oc_run(["get", "crd", TENANT_CRD]) if r.returncode != 0: if _oc_output_not_found(r): pytest.skip( f"Missing CRD {TENANT_CRD} (transitional skip: install maas-controller manifests " f"so CRDs exist; then controller creates {TENANT_NAME})." ) combined = (r.stderr or "") + (r.stdout or "") > pytest.fail(f"`oc get crd {TENANT_CRD}` failed: {combined.strip()}") E Failed: `oc get crd maastenantconfigs.maas.opendatahub.io` failed: Unable to connect to the server: dial tcp: lookup a1ae40bdd1dc145c69148e4a7dfcf45a-951b2e8c75f8896d.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host test/e2e/tests/test_tenant.py:95: Failed/workspace/source/test/e2e/tests/test_tenant_external_model_compat.py:377: legacy IPP coexistence E2E requires the MaaS IPP image pin containing the legacy-controller exclusion guard; set TENANT_EXTERNAL_MODEL_LEGACY_IPP_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_compat.py:386: legacy IPP coexistence E2E requires the MaaS IPP image pin containing the legacy-controller exclusion guard; set TENANT_EXTERNAL_MODEL_LEGACY_IPP_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_compat.py:401: legacy IPP coexistence E2E requires the MaaS IPP image pin containing the legacy-controller exclusion guard; set TENANT_EXTERNAL_MODEL_LEGACY_IPP_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_compat.py:411: legacy IPP coexistence E2E requires the MaaS IPP image pin containing the legacy-controller exclusion guard; set TENANT_EXTERNAL_MODEL_LEGACY_IPP_E2E=true to enable/workspace/source/test/e2e/tests/test_tenant_external_model_compat.py:425: legacy IPP coexistence E2E requires the MaaS IPP image pin containing the legacy-controller exclusion guard; set TENANT_EXTERNAL_MODEL_LEGACY_IPP_E2E=true to enable@pytest.fixture(scope="module", autouse=True) def _require_multitenancy_prerequisites(): > require_tenant_namespace_discovery() test/e2e/tests/test_tenant_namespace_discovery.py:73: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ def require_tenant_namespace_discovery(): if env_bool("ENABLE_TENANT_NAMESPACE_DISCOVERY"): if not controller_has_tenant_namespace_discovery(): > pytest.fail( "ENABLE_TENANT_NAMESPACE_DISCOVERY=true but maas-controller is missing " f"{DISCOVERY_ARG}; patch the deployment or run prow with multitenancy setup" ) E Failed: ENABLE_TENANT_NAMESPACE_DISCOVERY=true but maas-controller is missing --enable-tenant-namespace-discovery=true; patch the deployment or run prow with multitenancy setup test/e2e/tests/multitenancy_helpers.py:530: Failedself = <test_trlp_rate_grouping.TestRateGrouping object at 0x7fecd6d59050> cleanup = _Created(crs=[], sas=['e2e-rg-same-7d8230'], keys=[]) def test_same_rate_shares_limit_not_budget(self, cleanup): """Two subscriptions at one rate share a single TRLP limit, yet exhausting one leaves the same user's key on the other untouched. This is the core regression: with counters on userid alone, one user holding both subscriptions would drain both from one budget. """ ns = _ns() s = _suffix() rate = (LOW_TOKEN_LIMIT, EXHAUST_WINDOW) name = _limit_name(rate) sub_a, sub_b = f"e2e-rg-same-a-{s}", f"e2e-rg-same-b-{s}" > oc_token, user = _new_sa(cleanup, "e2e-rg-same") ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/tests/test_trlp_rate_grouping.py:543: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_trlp_rate_grouping.py:468: in _new_sa token = _create_sa_token(name, namespace=_ns(), duration=SA_TOKEN_DURATION) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ sa_name = 'e2e-rg-same-7d8230', namespace = 'models-as-a-service' duration = '20m' def _create_sa_token(sa_name, namespace=None, duration="10m"): namespace = namespace or _ns() sa_result = subprocess.run( ["oc", "create", "sa", sa_name, "-n", namespace], capture_output=True, text=True ) if sa_result.returncode != 0 and "already exists" not in sa_result.stderr: > raise RuntimeError(f"Failed to create SA {sa_name}: {sa_result.stderr}") E RuntimeError: Failed to create SA e2e-rg-same-7d8230: error: failed to create serviceaccount: Post "https://a1ae40bdd1dc145c69148e4a7dfcf45a-951b2e8c75f8896d.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/models-as-a-service/serviceaccounts?fieldManager=kubectl-create&fieldValidation=Ignore": dial tcp: lookup a1ae40bdd1dc145c69148e4a7dfcf45a-951b2e8c75f8896d.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host test/e2e/tests/test_helper.py:517: RuntimeErrorself = <test_trlp_rate_grouping.TestRateGrouping object at 0x7fecd6d59990> cleanup = _Created(crs=[], sas=['e2e-rg-user1-b3a81e'], keys=[]) def test_users_on_same_subscription_are_independent(self, cleanup): """Two users on one subscription each get the full budget.""" s = _suffix() rate = (LOW_TOKEN_LIMIT, EXHAUST_WINDOW) sub = f"e2e-rg-users-{s}" > token_1, user_1 = _new_sa(cleanup, "e2e-rg-user1") ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/tests/test_trlp_rate_grouping.py:576: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_trlp_rate_grouping.py:468: in _new_sa token = _create_sa_token(name, namespace=_ns(), duration=SA_TOKEN_DURATION) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ sa_name = 'e2e-rg-user1-b3a81e', namespace = 'models-as-a-service' duration = '20m' def _create_sa_token(sa_name, namespace=None, duration="10m"): namespace = namespace or _ns() sa_result = subprocess.run( ["oc", "create", "sa", sa_name, "-n", namespace], capture_output=True, text=True ) if sa_result.returncode != 0 and "already exists" not in sa_result.stderr: > raise RuntimeError(f"Failed to create SA {sa_name}: {sa_result.stderr}") E RuntimeError: Failed to create SA e2e-rg-user1-b3a81e: error: failed to create serviceaccount: Post "https://a1ae40bdd1dc145c69148e4a7dfcf45a-951b2e8c75f8896d.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/models-as-a-service/serviceaccounts?fieldManager=kubectl-create&fieldValidation=Ignore": dial tcp: lookup a1ae40bdd1dc145c69148e4a7dfcf45a-951b2e8c75f8896d.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host test/e2e/tests/test_helper.py:517: RuntimeErrorself = <test_trlp_rate_grouping.TestRateGrouping object at 0x7fecd6d55c50> cleanup = _Created(crs=[], sas=['e2e-rg-diff-f8ae36'], keys=[]) def test_different_rates_get_separate_limits(self, cleanup): """Two rates on one model produce two limits, each enforced at its own budget.""" s = _suffix() low, high = (LOW_TOKEN_LIMIT, EXHAUST_WINDOW), (LOW_TOKEN_LIMIT * 4, EXHAUST_WINDOW) low_name, high_name = _limit_name(low), _limit_name(high) sub_low, sub_high = f"e2e-rg-low-{s}", f"e2e-rg-high-{s}" > oc_token, user = _new_sa(cleanup, "e2e-rg-diff") ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/tests/test_trlp_rate_grouping.py:602: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_trlp_rate_grouping.py:468: in _new_sa token = _create_sa_token(name, namespace=_ns(), duration=SA_TOKEN_DURATION) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ sa_name = 'e2e-rg-diff-f8ae36', namespace = 'models-as-a-service' duration = '20m' def _create_sa_token(sa_name, namespace=None, duration="10m"): namespace = namespace or _ns() sa_result = subprocess.run( ["oc", "create", "sa", sa_name, "-n", namespace], capture_output=True, text=True ) if sa_result.returncode != 0 and "already exists" not in sa_result.stderr: > raise RuntimeError(f"Failed to create SA {sa_name}: {sa_result.stderr}") E RuntimeError: Failed to create SA e2e-rg-diff-f8ae36: error: failed to create serviceaccount: Post "https://a1ae40bdd1dc145c69148e4a7dfcf45a-951b2e8c75f8896d.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/models-as-a-service/serviceaccounts?fieldManager=kubectl-create&fieldValidation=Ignore": dial tcp: lookup a1ae40bdd1dc145c69148e4a7dfcf45a-951b2e8c75f8896d.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host test/e2e/tests/test_helper.py:517: RuntimeErrorself = <test_trlp_rate_grouping.TestRateGrouping object at 0x7fecd6d59f50> cleanup = _Created(crs=[], sas=['e2e-rg-edit-86ce08'], keys=[]) def test_rate_edit_moves_subscription_between_limits(self, cleanup): """Editing a live subscription's rate moves its clause out of the shared limit into the limit for the new rate, and its key stays rate limited. The key's selected_subscription_id does not change with the rate, and the TRLP update moves the clause in one write, so each request is counted by the old limit or the new one, never by neither. """ s = _suffix() old_rate = (LOW_TOKEN_LIMIT, EXHAUST_WINDOW) new_rate = (LOW_TOKEN_LIMIT + 6, EXHAUST_WINDOW) old_name, new_name = _limit_name(old_rate), _limit_name(new_rate) sub_a, sub_b = f"e2e-rg-edit-a-{s}", f"e2e-rg-edit-b-{s}" > oc_token, user = _new_sa(cleanup, "e2e-rg-edit") ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/tests/test_trlp_rate_grouping.py:642: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_trlp_rate_grouping.py:468: in _new_sa token = _create_sa_token(name, namespace=_ns(), duration=SA_TOKEN_DURATION) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ sa_name = 'e2e-rg-edit-86ce08', namespace = 'models-as-a-service' duration = '20m' def _create_sa_token(sa_name, namespace=None, duration="10m"): namespace = namespace or _ns() sa_result = subprocess.run( ["oc", "create", "sa", sa_name, "-n", namespace], capture_output=True, text=True ) if sa_result.returncode != 0 and "already exists" not in sa_result.stderr: > raise RuntimeError(f"Failed to create SA {sa_name}: {sa_result.stderr}") E RuntimeError: Failed to create SA e2e-rg-edit-86ce08: error: failed to create serviceaccount: Post "https://a1ae40bdd1dc145c69148e4a7dfcf45a-951b2e8c75f8896d.elb.us-east-2.amazonaws.com:6443/api/v1/namespaces/models-as-a-service/serviceaccounts?fieldManager=kubectl-create&fieldValidation=Ignore": dial tcp: lookup a1ae40bdd1dc145c69148e4a7dfcf45a-951b2e8c75f8896d.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host test/e2e/tests/test_helper.py:517: RuntimeErrorself = <test_trlp_rate_grouping.TestRateGrouping object at 0x7fecd6d5a550> cleanup = _Created(crs=[('maassubscription', 'e2e-rg-size-base-23b75f')], sas=[], keys=[]) def test_gateway_wasm_config_grows_by_clause_not_limit(self, cleanup): """Subscriptions at an existing rate add predicate clauses to the gateway's wasm config, not limits (RHOAIENG-95277). Kuadrant renders every TRLP limit as two wasm actions in each route-match ActionSet of kuadrant-<gateway>. With one limit per subscription each (subscription, model) pair cost ~27 KB per listener and the object hit etcd's size limit. A clause costs ~3 KB per listener at 19 route matches, so MAX_GROWTH_PER_SUBSCRIPTION sits well below a full limit. A new rate still adds exactly one limit. """ s = _suffix() window = "1h" # Random rates keep objects leaked by a failed run out of this run's limits. existing_rate = (5000 + random.randrange(100000), window) new_rate = (existing_rate[0] + 1, window) existing_name, new_name = _limit_name(existing_rate), _limit_name(new_rate) # Nobody calls the model here, so the owner needs no service account: # the controller renders limits from the subscription spec alone. owner = [f"system:serviceaccount:{_ns()}:e2e-rg-size-nobody"] base_sub = f"e2e-rg-size-base-{s}" > _add_subscription(cleanup, base_sub, owner, *existing_rate) test/e2e/tests/test_trlp_rate_grouping.py:692: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/test_trlp_rate_grouping.py:445: in _add_subscription _create_test_subscription(name=name, model_refs=[MODEL_REF], users=users, token_limit=limit, window=window) test/e2e/tests/test_helper.py:1298: in _create_test_subscription _apply_cr( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ cr_dict = {'apiVersion': 'maas.opendatahub.io/v1alpha1', 'kind': 'MaaSSubscription', 'metadata': {'name': 'e2e-rg-size-base-23b7...nconfigured-facebook-opt-125m-simulated', 'namespace': 'llm', 'tokenRateLimits': [{'limit': 39836, 'window': '1h'}]}]}} def _apply_cr(cr_dict): result = _run_oc(["oc", "apply", "-f", "-"], input_text=json.dumps(cr_dict)) if result.returncode != 0: meta = cr_dict.get("metadata", {}) > raise RuntimeError( f"`oc apply` failed for {cr_dict.get('kind')} " f"{meta.get('namespace', '<cluster>')}/{meta.get('name')}: " f"{result.stderr.strip() or result.stdout.strip()}" ) E RuntimeError: `oc apply` failed for MaaSSubscription models-as-a-service/e2e-rg-size-base-23b75f: error: error when retrieving current configuration of: E Resource: "maas.opendatahub.io/v1alpha1, Resource=maassubscriptions", GroupVersionKind: "maas.opendatahub.io/v1alpha1, Kind=MaaSSubscription" E Name: "e2e-rg-size-base-23b75f", Namespace: "models-as-a-service" E from server for: "STDIN": Get "https://a1ae40bdd1dc145c69148e4a7dfcf45a-951b2e8c75f8896d.elb.us-east-2.amazonaws.com:6443/apis/maas.opendatahub.io/v1alpha1/namespaces/models-as-a-service/maassubscriptions/e2e-rg-size-base-23b75f": dial tcp: lookup a1ae40bdd1dc145c69148e4a7dfcf45a-951b2e8c75f8896d.elb.us-east-2.amazonaws.com on 172.30.0.10:53: no such host test/e2e/tests/test_helper.py:894: RuntimeErrorself = <HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443) at 0x7fecd598e290> def _new_conn(self) -> socket.socket: """Establish a socket connection and set nodelay settings on it. :return: New socket connection. """ try: > sock = connection.create_connection( (self._dns_host, self.port), self.timeout, source_address=self.source_address, socket_options=self.socket_options, ) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connection.py:239: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.11/site-packages/urllib3/util/connection.py:60: in create_connection for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM): ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host = 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port = 443 family = <AddressFamily.AF_UNSPEC: 0>, type = <SocketKind.SOCK_STREAM: 1> proto = 0, flags = 0 def getaddrinfo(host, port, family=0, type=0, proto=0, flags=0): """Resolve host and port into list of address info entries. Translate the host/port argument into a sequence of 5-tuples that contain all the necessary arguments for creating a socket connected to that service. host is a domain name, a string representation of an IPv4/v6 address or None. port is a string service name such as 'http', a numeric port number or None. By passing None as the value of host and port, you can pass NULL to the underlying C API. The family, type and proto arguments can be optionally specified in order to narrow the list of addresses returned. Passing zero as a value for each of these arguments selects the full range of results. """ # We override this function since we want to translate the numeric family # and socket type values to enum constants. addrlist = [] > for res in _socket.getaddrinfo(host, port, family, type, proto, flags): ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E socket.gaierror: [Errno -2] Name or service not known /usr/lib64/python3.11/socket.py:974: gaierror The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fecd67b8990> method = 'POST', url = '/maas-api/v1/api-keys' body = b'{"subscription": "simulator-subscription", "name": "e2e-test-inference-key-main-15ba0a"}' headers = {'User-Agent': 'python-requests/2.34.2', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-aliv...0H7rToc9Lr08DFKjBYosc3tfxNWKEVBC39-r2YYV90wxoYoSn6fNtCqMw', 'Content-Type': 'application/json', 'Content-Length': '88'} retries = Retry(total=0, connect=None, read=False, redirect=None, status=None) redirect = False, assert_same_host = False timeout = Timeout(connect=30, read=30, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): # URLs starting with / are inherently schemeless. url = to_str(_encode_target(url)) destination_scheme = None else: parsed_url = parse_url(url) destination_scheme = parsed_url.scheme url = to_str(parsed_url._replace(fragment=None).url) if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) timeout_obj = self._get_timeout(timeout) try: # Request a connection from the queue. conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:793: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:494: in _make_request raise new_e test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:470: in _make_request self._validate_conn(conn) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1125: in _validate_conn conn.connect() test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connection.py:827: in connect self.sock = sock = self._new_conn() ^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443) at 0x7fecd598e290> def _new_conn(self) -> socket.socket: """Establish a socket connection and set nodelay settings on it. :return: New socket connection. """ try: sock = connection.create_connection( (self._dns_host, self.port), self.timeout, source_address=self.source_address, socket_options=self.socket_options, ) except socket.gaierror as e: > raise NameResolutionError(self.host, self, e) from e E urllib3.exceptions.NameResolutionError: HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Failed to resolve 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev' ([Errno -2] Name or service not known) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connection.py:246: NameResolutionError The above exception was the direct cause of the following exception: self = <requests.adapters.HTTPAdapter object at 0x7fecd5b7b790> request = <PreparedRequest [POST]>, stream = False, timeout = 30, verify = False cert = None, proxies = OrderedDict() def send( self, request: PreparedRequest, stream: bool = False, timeout: _t.TimeoutType = None, verify: _t.VerifyType = True, cert: _t.CertType = None, proxies: dict[str, str] | None = None, ) -> Response: """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ assert _is_prepared(request) try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout resolved_timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): resolved_timeout = timeout else: resolved_timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str] headers=request.headers, # type: ignore[arg-type] # urllib3#3072 redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=resolved_timeout, chunked=chunked, ) test/e2e/.venv/lib64/python3.11/site-packages/requests/adapters.py:696: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:847: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=None, read=False, redirect=None, status=None) method = 'POST', url = '/maas-api/v1/api-keys', response = None error = NameResolutionError("HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Failed to resolve 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev' ([Errno -2] Name or service not known)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fecd67b8990> _stacktrace = <traceback object at 0x7fecd598c300> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Max retries exceeded with url: /maas-api/v1/api-keys (Caused by NameResolutionError("HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Failed to resolve 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev' ([Errno -2] Name or service not known)")) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/util/retry.py:555: MaxRetryError During handling of the above exception, another exception occurred: api_keys_base_url = 'https://maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev/maas-api/v1/api-keys' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6ImJYeEpEQkYwenZKdUJabFlVai02RWk4Mlkwck50RDkwVG1MOWRxY3R4VnMifQ.e...yeXTv-V-ZsjcVoIX7C4JiRZS0H7rToc9Lr08DFKjBYosc3tfxNWKEVBC39-r2YYV90wxoYoSn6fNtCqMw', 'Content-Type': 'application/json'} @pytest.fixture(scope="module") def api_key(api_keys_base_url: str, headers: dict) -> str: """ Create an API key for model inference tests. Returns the plaintext API key (show-once pattern). Note: The key inherits the authenticated user's groups, which should include system:authenticated to satisfy AuthPolicy requirements for model access. """ from multitenancy_helpers import response_summary sim_sub = os.environ.get("E2E_SIMULATOR_SUBSCRIPTION", "simulator-subscription") key_name = f"e2e-test-inference-key-{_xdist_worker_suffix()}" print(f"[api_key] Creating API key for inference tests ({key_name})...") > r = _create_api_key( api_keys_base_url, headers, {"subscription": sim_sub}, key_name, ) test/e2e/tests/conftest.py:426: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/conftest.py:311: in _create_api_key response = requests.post(url, headers=headers, json=body, timeout=30, verify=TLS_VERIFY) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/.venv/lib64/python3.11/site-packages/requests/api.py:134: in post return request("post", url, data=data, json=json, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/.venv/lib64/python3.11/site-packages/requests/api.py:71: in request return session.request(method=method, url=url, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/.venv/lib64/python3.11/site-packages/requests/sessions.py:651: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/.venv/lib64/python3.11/site-packages/requests/sessions.py:784: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <requests.adapters.HTTPAdapter object at 0x7fecd5b7b790> request = <PreparedRequest [POST]>, stream = False, timeout = 30, verify = False cert = None, proxies = OrderedDict() def send( self, request: PreparedRequest, stream: bool = False, timeout: _t.TimeoutType = None, verify: _t.VerifyType = True, cert: _t.CertType = None, proxies: dict[str, str] | None = None, ) -> Response: """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ assert _is_prepared(request) try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout resolved_timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): resolved_timeout = timeout else: resolved_timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str] headers=request.headers, # type: ignore[arg-type] # urllib3#3072 redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=resolved_timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Max retries exceeded with url: /maas-api/v1/api-keys (Caused by NameResolutionError("HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Failed to resolve 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev' ([Errno -2] Name or service not known)")) test/e2e/.venv/lib64/python3.11/site-packages/requests/adapters.py:729: ConnectionErrorself = <HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443) at 0x7fecd598e290> def _new_conn(self) -> socket.socket: """Establish a socket connection and set nodelay settings on it. :return: New socket connection. """ try: > sock = connection.create_connection( (self._dns_host, self.port), self.timeout, source_address=self.source_address, socket_options=self.socket_options, ) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connection.py:239: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.11/site-packages/urllib3/util/connection.py:60: in create_connection for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM): ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host = 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port = 443 family = <AddressFamily.AF_UNSPEC: 0>, type = <SocketKind.SOCK_STREAM: 1> proto = 0, flags = 0 def getaddrinfo(host, port, family=0, type=0, proto=0, flags=0): """Resolve host and port into list of address info entries. Translate the host/port argument into a sequence of 5-tuples that contain all the necessary arguments for creating a socket connected to that service. host is a domain name, a string representation of an IPv4/v6 address or None. port is a string service name such as 'http', a numeric port number or None. By passing None as the value of host and port, you can pass NULL to the underlying C API. The family, type and proto arguments can be optionally specified in order to narrow the list of addresses returned. Passing zero as a value for each of these arguments selects the full range of results. """ # We override this function since we want to translate the numeric family # and socket type values to enum constants. addrlist = [] > for res in _socket.getaddrinfo(host, port, family, type, proto, flags): ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E socket.gaierror: [Errno -2] Name or service not known /usr/lib64/python3.11/socket.py:974: gaierror The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fecd67b8990> method = 'POST', url = '/maas-api/v1/api-keys' body = b'{"subscription": "simulator-subscription", "name": "e2e-test-inference-key-main-15ba0a"}' headers = {'User-Agent': 'python-requests/2.34.2', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-aliv...0H7rToc9Lr08DFKjBYosc3tfxNWKEVBC39-r2YYV90wxoYoSn6fNtCqMw', 'Content-Type': 'application/json', 'Content-Length': '88'} retries = Retry(total=0, connect=None, read=False, redirect=None, status=None) redirect = False, assert_same_host = False timeout = Timeout(connect=30, read=30, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): # URLs starting with / are inherently schemeless. url = to_str(_encode_target(url)) destination_scheme = None else: parsed_url = parse_url(url) destination_scheme = parsed_url.scheme url = to_str(parsed_url._replace(fragment=None).url) if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) timeout_obj = self._get_timeout(timeout) try: # Request a connection from the queue. conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:793: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:494: in _make_request raise new_e test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:470: in _make_request self._validate_conn(conn) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1125: in _validate_conn conn.connect() test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connection.py:827: in connect self.sock = sock = self._new_conn() ^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443) at 0x7fecd598e290> def _new_conn(self) -> socket.socket: """Establish a socket connection and set nodelay settings on it. :return: New socket connection. """ try: sock = connection.create_connection( (self._dns_host, self.port), self.timeout, source_address=self.source_address, socket_options=self.socket_options, ) except socket.gaierror as e: > raise NameResolutionError(self.host, self, e) from e E urllib3.exceptions.NameResolutionError: HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Failed to resolve 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev' ([Errno -2] Name or service not known) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connection.py:246: NameResolutionError The above exception was the direct cause of the following exception: self = <requests.adapters.HTTPAdapter object at 0x7fecd5b7b790> request = <PreparedRequest [POST]>, stream = False, timeout = 30, verify = False cert = None, proxies = OrderedDict() def send( self, request: PreparedRequest, stream: bool = False, timeout: _t.TimeoutType = None, verify: _t.VerifyType = True, cert: _t.CertType = None, proxies: dict[str, str] | None = None, ) -> Response: """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ assert _is_prepared(request) try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout resolved_timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): resolved_timeout = timeout else: resolved_timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str] headers=request.headers, # type: ignore[arg-type] # urllib3#3072 redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=resolved_timeout, chunked=chunked, ) test/e2e/.venv/lib64/python3.11/site-packages/requests/adapters.py:696: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:847: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=None, read=False, redirect=None, status=None) method = 'POST', url = '/maas-api/v1/api-keys', response = None error = NameResolutionError("HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Failed to resolve 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev' ([Errno -2] Name or service not known)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fecd67b8990> _stacktrace = <traceback object at 0x7fecd598c300> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Max retries exceeded with url: /maas-api/v1/api-keys (Caused by NameResolutionError("HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Failed to resolve 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev' ([Errno -2] Name or service not known)")) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/util/retry.py:555: MaxRetryError During handling of the above exception, another exception occurred: api_keys_base_url = 'https://maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev/maas-api/v1/api-keys' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6ImJYeEpEQkYwenZKdUJabFlVai02RWk4Mlkwck50RDkwVG1MOWRxY3R4VnMifQ.e...yeXTv-V-ZsjcVoIX7C4JiRZS0H7rToc9Lr08DFKjBYosc3tfxNWKEVBC39-r2YYV90wxoYoSn6fNtCqMw', 'Content-Type': 'application/json'} @pytest.fixture(scope="module") def api_key(api_keys_base_url: str, headers: dict) -> str: """ Create an API key for model inference tests. Returns the plaintext API key (show-once pattern). Note: The key inherits the authenticated user's groups, which should include system:authenticated to satisfy AuthPolicy requirements for model access. """ from multitenancy_helpers import response_summary sim_sub = os.environ.get("E2E_SIMULATOR_SUBSCRIPTION", "simulator-subscription") key_name = f"e2e-test-inference-key-{_xdist_worker_suffix()}" print(f"[api_key] Creating API key for inference tests ({key_name})...") > r = _create_api_key( api_keys_base_url, headers, {"subscription": sim_sub}, key_name, ) test/e2e/tests/conftest.py:426: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/conftest.py:311: in _create_api_key response = requests.post(url, headers=headers, json=body, timeout=30, verify=TLS_VERIFY) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/.venv/lib64/python3.11/site-packages/requests/api.py:134: in post return request("post", url, data=data, json=json, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/.venv/lib64/python3.11/site-packages/requests/api.py:71: in request return session.request(method=method, url=url, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/.venv/lib64/python3.11/site-packages/requests/sessions.py:651: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/.venv/lib64/python3.11/site-packages/requests/sessions.py:784: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <requests.adapters.HTTPAdapter object at 0x7fecd5b7b790> request = <PreparedRequest [POST]>, stream = False, timeout = 30, verify = False cert = None, proxies = OrderedDict() def send( self, request: PreparedRequest, stream: bool = False, timeout: _t.TimeoutType = None, verify: _t.VerifyType = True, cert: _t.CertType = None, proxies: dict[str, str] | None = None, ) -> Response: """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ assert _is_prepared(request) try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout resolved_timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): resolved_timeout = timeout else: resolved_timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str] headers=request.headers, # type: ignore[arg-type] # urllib3#3072 redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=resolved_timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Max retries exceeded with url: /maas-api/v1/api-keys (Caused by NameResolutionError("HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Failed to resolve 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev' ([Errno -2] Name or service not known)")) test/e2e/.venv/lib64/python3.11/site-packages/requests/adapters.py:729: ConnectionErrorself = <HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443) at 0x7fecd598e290> def _new_conn(self) -> socket.socket: """Establish a socket connection and set nodelay settings on it. :return: New socket connection. """ try: > sock = connection.create_connection( (self._dns_host, self.port), self.timeout, source_address=self.source_address, socket_options=self.socket_options, ) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connection.py:239: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.11/site-packages/urllib3/util/connection.py:60: in create_connection for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM): ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host = 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port = 443 family = <AddressFamily.AF_UNSPEC: 0>, type = <SocketKind.SOCK_STREAM: 1> proto = 0, flags = 0 def getaddrinfo(host, port, family=0, type=0, proto=0, flags=0): """Resolve host and port into list of address info entries. Translate the host/port argument into a sequence of 5-tuples that contain all the necessary arguments for creating a socket connected to that service. host is a domain name, a string representation of an IPv4/v6 address or None. port is a string service name such as 'http', a numeric port number or None. By passing None as the value of host and port, you can pass NULL to the underlying C API. The family, type and proto arguments can be optionally specified in order to narrow the list of addresses returned. Passing zero as a value for each of these arguments selects the full range of results. """ # We override this function since we want to translate the numeric family # and socket type values to enum constants. addrlist = [] > for res in _socket.getaddrinfo(host, port, family, type, proto, flags): ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E socket.gaierror: [Errno -2] Name or service not known /usr/lib64/python3.11/socket.py:974: gaierror The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fecd67b8990> method = 'POST', url = '/maas-api/v1/api-keys' body = b'{"subscription": "simulator-subscription", "name": "e2e-test-inference-key-main-15ba0a"}' headers = {'User-Agent': 'python-requests/2.34.2', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-aliv...0H7rToc9Lr08DFKjBYosc3tfxNWKEVBC39-r2YYV90wxoYoSn6fNtCqMw', 'Content-Type': 'application/json', 'Content-Length': '88'} retries = Retry(total=0, connect=None, read=False, redirect=None, status=None) redirect = False, assert_same_host = False timeout = Timeout(connect=30, read=30, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): # URLs starting with / are inherently schemeless. url = to_str(_encode_target(url)) destination_scheme = None else: parsed_url = parse_url(url) destination_scheme = parsed_url.scheme url = to_str(parsed_url._replace(fragment=None).url) if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) timeout_obj = self._get_timeout(timeout) try: # Request a connection from the queue. conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:793: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:494: in _make_request raise new_e test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:470: in _make_request self._validate_conn(conn) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1125: in _validate_conn conn.connect() test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connection.py:827: in connect self.sock = sock = self._new_conn() ^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443) at 0x7fecd598e290> def _new_conn(self) -> socket.socket: """Establish a socket connection and set nodelay settings on it. :return: New socket connection. """ try: sock = connection.create_connection( (self._dns_host, self.port), self.timeout, source_address=self.source_address, socket_options=self.socket_options, ) except socket.gaierror as e: > raise NameResolutionError(self.host, self, e) from e E urllib3.exceptions.NameResolutionError: HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Failed to resolve 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev' ([Errno -2] Name or service not known) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connection.py:246: NameResolutionError The above exception was the direct cause of the following exception: self = <requests.adapters.HTTPAdapter object at 0x7fecd5b7b790> request = <PreparedRequest [POST]>, stream = False, timeout = 30, verify = False cert = None, proxies = OrderedDict() def send( self, request: PreparedRequest, stream: bool = False, timeout: _t.TimeoutType = None, verify: _t.VerifyType = True, cert: _t.CertType = None, proxies: dict[str, str] | None = None, ) -> Response: """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ assert _is_prepared(request) try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout resolved_timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): resolved_timeout = timeout else: resolved_timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str] headers=request.headers, # type: ignore[arg-type] # urllib3#3072 redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=resolved_timeout, chunked=chunked, ) test/e2e/.venv/lib64/python3.11/site-packages/requests/adapters.py:696: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:847: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=None, read=False, redirect=None, status=None) method = 'POST', url = '/maas-api/v1/api-keys', response = None error = NameResolutionError("HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Failed to resolve 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev' ([Errno -2] Name or service not known)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fecd67b8990> _stacktrace = <traceback object at 0x7fecd598c300> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Max retries exceeded with url: /maas-api/v1/api-keys (Caused by NameResolutionError("HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Failed to resolve 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev' ([Errno -2] Name or service not known)")) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/util/retry.py:555: MaxRetryError During handling of the above exception, another exception occurred: api_keys_base_url = 'https://maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev/maas-api/v1/api-keys' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6ImJYeEpEQkYwenZKdUJabFlVai02RWk4Mlkwck50RDkwVG1MOWRxY3R4VnMifQ.e...yeXTv-V-ZsjcVoIX7C4JiRZS0H7rToc9Lr08DFKjBYosc3tfxNWKEVBC39-r2YYV90wxoYoSn6fNtCqMw', 'Content-Type': 'application/json'} @pytest.fixture(scope="module") def api_key(api_keys_base_url: str, headers: dict) -> str: """ Create an API key for model inference tests. Returns the plaintext API key (show-once pattern). Note: The key inherits the authenticated user's groups, which should include system:authenticated to satisfy AuthPolicy requirements for model access. """ from multitenancy_helpers import response_summary sim_sub = os.environ.get("E2E_SIMULATOR_SUBSCRIPTION", "simulator-subscription") key_name = f"e2e-test-inference-key-{_xdist_worker_suffix()}" print(f"[api_key] Creating API key for inference tests ({key_name})...") > r = _create_api_key( api_keys_base_url, headers, {"subscription": sim_sub}, key_name, ) test/e2e/tests/conftest.py:426: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/conftest.py:311: in _create_api_key response = requests.post(url, headers=headers, json=body, timeout=30, verify=TLS_VERIFY) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/.venv/lib64/python3.11/site-packages/requests/api.py:134: in post return request("post", url, data=data, json=json, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/.venv/lib64/python3.11/site-packages/requests/api.py:71: in request return session.request(method=method, url=url, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/.venv/lib64/python3.11/site-packages/requests/sessions.py:651: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/.venv/lib64/python3.11/site-packages/requests/sessions.py:784: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <requests.adapters.HTTPAdapter object at 0x7fecd5b7b790> request = <PreparedRequest [POST]>, stream = False, timeout = 30, verify = False cert = None, proxies = OrderedDict() def send( self, request: PreparedRequest, stream: bool = False, timeout: _t.TimeoutType = None, verify: _t.VerifyType = True, cert: _t.CertType = None, proxies: dict[str, str] | None = None, ) -> Response: """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ assert _is_prepared(request) try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout resolved_timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): resolved_timeout = timeout else: resolved_timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str] headers=request.headers, # type: ignore[arg-type] # urllib3#3072 redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=resolved_timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Max retries exceeded with url: /maas-api/v1/api-keys (Caused by NameResolutionError("HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Failed to resolve 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev' ([Errno -2] Name or service not known)")) test/e2e/.venv/lib64/python3.11/site-packages/requests/adapters.py:729: ConnectionErrorself = <HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443) at 0x7fecd598e290> def _new_conn(self) -> socket.socket: """Establish a socket connection and set nodelay settings on it. :return: New socket connection. """ try: > sock = connection.create_connection( (self._dns_host, self.port), self.timeout, source_address=self.source_address, socket_options=self.socket_options, ) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connection.py:239: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.11/site-packages/urllib3/util/connection.py:60: in create_connection for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM): ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host = 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port = 443 family = <AddressFamily.AF_UNSPEC: 0>, type = <SocketKind.SOCK_STREAM: 1> proto = 0, flags = 0 def getaddrinfo(host, port, family=0, type=0, proto=0, flags=0): """Resolve host and port into list of address info entries. Translate the host/port argument into a sequence of 5-tuples that contain all the necessary arguments for creating a socket connected to that service. host is a domain name, a string representation of an IPv4/v6 address or None. port is a string service name such as 'http', a numeric port number or None. By passing None as the value of host and port, you can pass NULL to the underlying C API. The family, type and proto arguments can be optionally specified in order to narrow the list of addresses returned. Passing zero as a value for each of these arguments selects the full range of results. """ # We override this function since we want to translate the numeric family # and socket type values to enum constants. addrlist = [] > for res in _socket.getaddrinfo(host, port, family, type, proto, flags): ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E socket.gaierror: [Errno -2] Name or service not known /usr/lib64/python3.11/socket.py:974: gaierror The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fecd67b8990> method = 'POST', url = '/maas-api/v1/api-keys' body = b'{"subscription": "simulator-subscription", "name": "e2e-test-inference-key-main-15ba0a"}' headers = {'User-Agent': 'python-requests/2.34.2', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-aliv...0H7rToc9Lr08DFKjBYosc3tfxNWKEVBC39-r2YYV90wxoYoSn6fNtCqMw', 'Content-Type': 'application/json', 'Content-Length': '88'} retries = Retry(total=0, connect=None, read=False, redirect=None, status=None) redirect = False, assert_same_host = False timeout = Timeout(connect=30, read=30, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): # URLs starting with / are inherently schemeless. url = to_str(_encode_target(url)) destination_scheme = None else: parsed_url = parse_url(url) destination_scheme = parsed_url.scheme url = to_str(parsed_url._replace(fragment=None).url) if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) timeout_obj = self._get_timeout(timeout) try: # Request a connection from the queue. conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:793: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:494: in _make_request raise new_e test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:470: in _make_request self._validate_conn(conn) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1125: in _validate_conn conn.connect() test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connection.py:827: in connect self.sock = sock = self._new_conn() ^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443) at 0x7fecd598e290> def _new_conn(self) -> socket.socket: """Establish a socket connection and set nodelay settings on it. :return: New socket connection. """ try: sock = connection.create_connection( (self._dns_host, self.port), self.timeout, source_address=self.source_address, socket_options=self.socket_options, ) except socket.gaierror as e: > raise NameResolutionError(self.host, self, e) from e E urllib3.exceptions.NameResolutionError: HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Failed to resolve 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev' ([Errno -2] Name or service not known) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connection.py:246: NameResolutionError The above exception was the direct cause of the following exception: self = <requests.adapters.HTTPAdapter object at 0x7fecd5b7b790> request = <PreparedRequest [POST]>, stream = False, timeout = 30, verify = False cert = None, proxies = OrderedDict() def send( self, request: PreparedRequest, stream: bool = False, timeout: _t.TimeoutType = None, verify: _t.VerifyType = True, cert: _t.CertType = None, proxies: dict[str, str] | None = None, ) -> Response: """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ assert _is_prepared(request) try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout resolved_timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): resolved_timeout = timeout else: resolved_timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str] headers=request.headers, # type: ignore[arg-type] # urllib3#3072 redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=resolved_timeout, chunked=chunked, ) test/e2e/.venv/lib64/python3.11/site-packages/requests/adapters.py:696: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:847: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=None, read=False, redirect=None, status=None) method = 'POST', url = '/maas-api/v1/api-keys', response = None error = NameResolutionError("HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Failed to resolve 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev' ([Errno -2] Name or service not known)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fecd67b8990> _stacktrace = <traceback object at 0x7fecd598c300> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Max retries exceeded with url: /maas-api/v1/api-keys (Caused by NameResolutionError("HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Failed to resolve 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev' ([Errno -2] Name or service not known)")) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/util/retry.py:555: MaxRetryError During handling of the above exception, another exception occurred: api_keys_base_url = 'https://maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev/maas-api/v1/api-keys' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6ImJYeEpEQkYwenZKdUJabFlVai02RWk4Mlkwck50RDkwVG1MOWRxY3R4VnMifQ.e...yeXTv-V-ZsjcVoIX7C4JiRZS0H7rToc9Lr08DFKjBYosc3tfxNWKEVBC39-r2YYV90wxoYoSn6fNtCqMw', 'Content-Type': 'application/json'} @pytest.fixture(scope="module") def api_key(api_keys_base_url: str, headers: dict) -> str: """ Create an API key for model inference tests. Returns the plaintext API key (show-once pattern). Note: The key inherits the authenticated user's groups, which should include system:authenticated to satisfy AuthPolicy requirements for model access. """ from multitenancy_helpers import response_summary sim_sub = os.environ.get("E2E_SIMULATOR_SUBSCRIPTION", "simulator-subscription") key_name = f"e2e-test-inference-key-{_xdist_worker_suffix()}" print(f"[api_key] Creating API key for inference tests ({key_name})...") > r = _create_api_key( api_keys_base_url, headers, {"subscription": sim_sub}, key_name, ) test/e2e/tests/conftest.py:426: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/conftest.py:311: in _create_api_key response = requests.post(url, headers=headers, json=body, timeout=30, verify=TLS_VERIFY) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/.venv/lib64/python3.11/site-packages/requests/api.py:134: in post return request("post", url, data=data, json=json, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/.venv/lib64/python3.11/site-packages/requests/api.py:71: in request return session.request(method=method, url=url, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/.venv/lib64/python3.11/site-packages/requests/sessions.py:651: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/.venv/lib64/python3.11/site-packages/requests/sessions.py:784: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <requests.adapters.HTTPAdapter object at 0x7fecd5b7b790> request = <PreparedRequest [POST]>, stream = False, timeout = 30, verify = False cert = None, proxies = OrderedDict() def send( self, request: PreparedRequest, stream: bool = False, timeout: _t.TimeoutType = None, verify: _t.VerifyType = True, cert: _t.CertType = None, proxies: dict[str, str] | None = None, ) -> Response: """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ assert _is_prepared(request) try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout resolved_timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): resolved_timeout = timeout else: resolved_timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str] headers=request.headers, # type: ignore[arg-type] # urllib3#3072 redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=resolved_timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Max retries exceeded with url: /maas-api/v1/api-keys (Caused by NameResolutionError("HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Failed to resolve 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev' ([Errno -2] Name or service not known)")) test/e2e/.venv/lib64/python3.11/site-packages/requests/adapters.py:729: ConnectionErrorself = <HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443) at 0x7fecd598e290> def _new_conn(self) -> socket.socket: """Establish a socket connection and set nodelay settings on it. :return: New socket connection. """ try: > sock = connection.create_connection( (self._dns_host, self.port), self.timeout, source_address=self.source_address, socket_options=self.socket_options, ) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connection.py:239: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.11/site-packages/urllib3/util/connection.py:60: in create_connection for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM): ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ host = 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port = 443 family = <AddressFamily.AF_UNSPEC: 0>, type = <SocketKind.SOCK_STREAM: 1> proto = 0, flags = 0 def getaddrinfo(host, port, family=0, type=0, proto=0, flags=0): """Resolve host and port into list of address info entries. Translate the host/port argument into a sequence of 5-tuples that contain all the necessary arguments for creating a socket connected to that service. host is a domain name, a string representation of an IPv4/v6 address or None. port is a string service name such as 'http', a numeric port number or None. By passing None as the value of host and port, you can pass NULL to the underlying C API. The family, type and proto arguments can be optionally specified in order to narrow the list of addresses returned. Passing zero as a value for each of these arguments selects the full range of results. """ # We override this function since we want to translate the numeric family # and socket type values to enum constants. addrlist = [] > for res in _socket.getaddrinfo(host, port, family, type, proto, flags): ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E socket.gaierror: [Errno -2] Name or service not known /usr/lib64/python3.11/socket.py:974: gaierror The above exception was the direct cause of the following exception: self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fecd67b8990> method = 'POST', url = '/maas-api/v1/api-keys' body = b'{"subscription": "simulator-subscription", "name": "e2e-test-inference-key-main-15ba0a"}' headers = {'User-Agent': 'python-requests/2.34.2', 'Accept-Encoding': 'gzip, deflate', 'Accept': '*/*', 'Connection': 'keep-aliv...0H7rToc9Lr08DFKjBYosc3tfxNWKEVBC39-r2YYV90wxoYoSn6fNtCqMw', 'Content-Type': 'application/json', 'Content-Length': '88'} retries = Retry(total=0, connect=None, read=False, redirect=None, status=None) redirect = False, assert_same_host = False timeout = Timeout(connect=30, read=30, total=None), pool_timeout = None release_conn = False, chunked = False, body_pos = None, preload_content = False decode_content = False, response_kw = {} def urlopen( # type: ignore[override] self, method: str, url: str, body: _TYPE_BODY | None = None, headers: typing.Mapping[str, str] | None = None, retries: Retry | bool | int | None = None, redirect: bool = True, assert_same_host: bool = True, timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT, pool_timeout: int | None = None, release_conn: bool | None = None, chunked: bool = False, body_pos: _TYPE_BODY_POSITION | None = None, preload_content: bool = True, decode_content: bool = True, **response_kw: typing.Any, ) -> BaseHTTPResponse: """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all the raw details. .. note:: More commonly, it's appropriate to use a convenience method such as :meth:`request`. .. note:: `release_conn` will only behave as expected if `preload_content=False` because we want to make `preload_content=False` the default behaviour someday soon without breaking backwards compatibility. :param method: HTTP request method (such as GET, POST, PUT, etc.) :param url: The URL to perform the request on. :param body: Data to send in the request body, either :class:`str`, :class:`bytes`, an iterable of :class:`str`/:class:`bytes`, or a file-like object. :param headers: Dictionary of custom headers to send, such as User-Agent, If-None-Match, etc. If None, pool headers are used. If provided, these headers completely replace any pool-specific headers. :param retries: Configure the number of retries to allow before raising a :class:`~urllib3.exceptions.MaxRetryError` exception. If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a :class:`~urllib3.util.retry.Retry` object for fine-grained control over different types of retries. Pass an integer number to retry connection errors that many times, but no other types of errors. Pass zero to never retry. If ``False``, then retries are disabled and any exception is raised immediately. Also, instead of raising a MaxRetryError on redirects, the redirect response will be returned. :type retries: :class:`~urllib3.util.retry.Retry`, False, or an int. :param redirect: If True, automatically handle redirects (status codes 301, 302, 303, 307, 308). Each redirect counts as a retry. Disabling retries will disable redirect, too. :param assert_same_host: If ``True``, will make sure that the host of the pool requests is consistent else will raise HostChangedError. When ``False``, you can use the pool on an HTTP proxy and request foreign hosts. :param timeout: If specified, overrides the default timeout for this one request. It may be a float (in seconds) or an instance of :class:`urllib3.util.Timeout`. :param pool_timeout: If set and the pool is set to block=True, then this method will block for ``pool_timeout`` seconds and raise EmptyPoolError if no connection is available within the time period. :param bool preload_content: If True, the response's body will be preloaded into memory. :param bool decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. :param release_conn: If False, then the urlopen call will not release the connection back into the pool once a response is received (but will release if you read the entire contents of the response such as when `preload_content=True`). This is useful if you're not preloading the response's content immediately. You will need to call ``r.release_conn()`` on the response ``r`` to return the connection back into the pool. If None, it takes the value of ``preload_content`` which defaults to ``True``. :param bool chunked: If True, urllib3 will send the body using chunked transfer encoding. Otherwise, urllib3 will send the body using the standard content-length form. Defaults to False. :param int body_pos: Position to seek to in file-like body in the event of a retry or redirect. Typically this won't need to be set because urllib3 will auto-populate the value when needed. """ # Ensure that the URL we're connecting to is properly encoded if url.startswith("/"): # URLs starting with / are inherently schemeless. url = to_str(_encode_target(url)) destination_scheme = None else: parsed_url = parse_url(url) destination_scheme = parsed_url.scheme url = to_str(parsed_url._replace(fragment=None).url) if headers is None: headers = self.headers if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: release_conn = preload_content # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) conn = None # Track whether `conn` needs to be released before # returning/raising/recursing. Update this variable if necessary, and # leave `release_conn` constant throughout the function. That way, if # the function recurses, the original value of `release_conn` will be # passed down into the recursive call, and its value will be respected. # # See issue #651 [1] for details. # # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn http_tunnel_required = connection_requires_http_tunnel( self.proxy, self.proxy_config, destination_scheme ) # Merge the proxy headers. Only done when not using HTTP CONNECT. We # have to copy the headers dict so we can safely change it without those # changes being reflected in anyone else's copy. if not http_tunnel_required: headers = headers.copy() # type: ignore[attr-defined] headers.update(self.proxy_headers) # type: ignore[union-attr] # Must keep the exception bound to a separate variable or else Python 3 # complains about UnboundLocalError. err = None # Keep track of whether we cleanly exited the except block. This # ensures we do proper cleanup in finally. clean_exit = False # Rewind body position, if needed. Record current position # for future rewinds in the event of a redirect/retry. body_pos = set_file_position(body, body_pos) timeout_obj = self._get_timeout(timeout) try: # Request a connection from the queue. conn = self._get_conn(timeout=pool_timeout) conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment] # Is this a closed/new connection that requires CONNECT tunnelling? if self.proxy is not None and http_tunnel_required and conn.is_closed: try: self._prepare_proxy(conn) except (BaseSSLError, OSError, SocketTimeout) as e: self._raise_timeout( err=e, url=self.proxy.url, timeout_value=conn.timeout ) raise # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise # it will also try to release it and we'll have a double-release # mess. response_conn = conn if not release_conn else None # Make the request on the HTTPConnection object > response = self._make_request( conn, method, url, timeout=timeout_obj, body=body, headers=headers, chunked=chunked, retries=retries, response_conn=response_conn, preload_content=preload_content, decode_content=decode_content, **response_kw, ) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:793: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:494: in _make_request raise new_e test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:470: in _make_request self._validate_conn(conn) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:1125: in _validate_conn conn.connect() test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connection.py:827: in connect self.sock = sock = self._new_conn() ^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443) at 0x7fecd598e290> def _new_conn(self) -> socket.socket: """Establish a socket connection and set nodelay settings on it. :return: New socket connection. """ try: sock = connection.create_connection( (self._dns_host, self.port), self.timeout, source_address=self.source_address, socket_options=self.socket_options, ) except socket.gaierror as e: > raise NameResolutionError(self.host, self, e) from e E urllib3.exceptions.NameResolutionError: HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Failed to resolve 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev' ([Errno -2] Name or service not known) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connection.py:246: NameResolutionError The above exception was the direct cause of the following exception: self = <requests.adapters.HTTPAdapter object at 0x7fecd5b7b790> request = <PreparedRequest [POST]>, stream = False, timeout = 30, verify = False cert = None, proxies = OrderedDict() def send( self, request: PreparedRequest, stream: bool = False, timeout: _t.TimeoutType = None, verify: _t.VerifyType = True, cert: _t.CertType = None, proxies: dict[str, str] | None = None, ) -> Response: """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ assert _is_prepared(request) try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout resolved_timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): resolved_timeout = timeout else: resolved_timeout = TimeoutSauce(connect=timeout, read=timeout) try: > resp = conn.urlopen( method=request.method, url=url, body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str] headers=request.headers, # type: ignore[arg-type] # urllib3#3072 redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=resolved_timeout, chunked=chunked, ) test/e2e/.venv/lib64/python3.11/site-packages/requests/adapters.py:696: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/.venv/lib64/python3.11/site-packages/urllib3/connectionpool.py:847: in urlopen retries = retries.increment( _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = Retry(total=0, connect=None, read=False, redirect=None, status=None) method = 'POST', url = '/maas-api/v1/api-keys', response = None error = NameResolutionError("HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Failed to resolve 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev' ([Errno -2] Name or service not known)") _pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fecd67b8990> _stacktrace = <traceback object at 0x7fecd598c300> def increment( self, method: str | None = None, url: str | None = None, response: BaseHTTPResponse | None = None, error: Exception | None = None, _pool: ConnectionPool | None = None, _stacktrace: TracebackType | None = None, ) -> Self: """Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not return a response. :type response: :class:`~urllib3.response.BaseHTTPResponse` :param Exception error: An error encountered during the request, or None if the response was received successfully. :return: A new ``Retry`` object. """ if self.total is False and error: # Disabled, indicate to re-raise the error. raise reraise(type(error), error, _stacktrace) total = self.total if total is not None: total -= 1 connect = self.connect read = self.read redirect = self.redirect status_count = self.status other = self.other cause = "unknown" status = None redirect_location = None if error and self._is_connection_error(error): # Connect retry? if connect is False: raise reraise(type(error), error, _stacktrace) elif connect is not None: connect -= 1 elif error and self._is_read_error(error): # Read retry? if read is False or method is None or not self._is_method_retryable(method): raise reraise(type(error), error, _stacktrace) elif read is not None: read -= 1 elif error: # Other retry? if other is not None: other -= 1 elif response and response.get_redirect_location(): # Redirect retry? if redirect is not None: redirect -= 1 cause = "too many redirects" response_redirect_location = response.get_redirect_location() if response_redirect_location: redirect_location = response_redirect_location status = response.status else: # Incrementing because of a server error like a 500 in # status_forcelist and the given method is in the allowed_methods cause = ResponseError.GENERIC_ERROR if response and response.status: if status_count is not None: status_count -= 1 cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status history = self.history + ( RequestHistory(method, url, error, status, redirect_location), ) new_retry = self.new( total=total, connect=connect, read=read, redirect=redirect, status=status_count, other=other, history=history, ) if new_retry.is_exhausted(): reason = error or ResponseError(cause) > raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Max retries exceeded with url: /maas-api/v1/api-keys (Caused by NameResolutionError("HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Failed to resolve 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev' ([Errno -2] Name or service not known)")) test/e2e/.venv/lib64/python3.11/site-packages/urllib3/util/retry.py:555: MaxRetryError During handling of the above exception, another exception occurred: api_keys_base_url = 'https://maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev/maas-api/v1/api-keys' headers = {'Authorization': 'Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6ImJYeEpEQkYwenZKdUJabFlVai02RWk4Mlkwck50RDkwVG1MOWRxY3R4VnMifQ.e...yeXTv-V-ZsjcVoIX7C4JiRZS0H7rToc9Lr08DFKjBYosc3tfxNWKEVBC39-r2YYV90wxoYoSn6fNtCqMw', 'Content-Type': 'application/json'} @pytest.fixture(scope="module") def api_key(api_keys_base_url: str, headers: dict) -> str: """ Create an API key for model inference tests. Returns the plaintext API key (show-once pattern). Note: The key inherits the authenticated user's groups, which should include system:authenticated to satisfy AuthPolicy requirements for model access. """ from multitenancy_helpers import response_summary sim_sub = os.environ.get("E2E_SIMULATOR_SUBSCRIPTION", "simulator-subscription") key_name = f"e2e-test-inference-key-{_xdist_worker_suffix()}" print(f"[api_key] Creating API key for inference tests ({key_name})...") > r = _create_api_key( api_keys_base_url, headers, {"subscription": sim_sub}, key_name, ) test/e2e/tests/conftest.py:426: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ test/e2e/tests/conftest.py:311: in _create_api_key response = requests.post(url, headers=headers, json=body, timeout=30, verify=TLS_VERIFY) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/.venv/lib64/python3.11/site-packages/requests/api.py:134: in post return request("post", url, data=data, json=json, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/.venv/lib64/python3.11/site-packages/requests/api.py:71: in request return session.request(method=method, url=url, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/.venv/lib64/python3.11/site-packages/requests/sessions.py:651: in request resp = self.send(prep, **send_kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ test/e2e/.venv/lib64/python3.11/site-packages/requests/sessions.py:784: in send r = adapter.send(request, **kwargs) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <requests.adapters.HTTPAdapter object at 0x7fecd5b7b790> request = <PreparedRequest [POST]>, stream = False, timeout = 30, verify = False cert = None, proxies = OrderedDict() def send( self, request: PreparedRequest, stream: bool = False, timeout: _t.TimeoutType = None, verify: _t.VerifyType = True, cert: _t.CertType = None, proxies: dict[str, str] | None = None, ) -> Response: """Sends PreparedRequest object. Returns Response object. :param request: The :class:`PreparedRequest <PreparedRequest>` being sent. :param stream: (optional) Whether to stream the request content. :param timeout: (optional) How long to wait for the server to send data before giving up, as a float, or a :ref:`(connect timeout, read timeout) <timeouts>` tuple. :type timeout: float or tuple or urllib3 Timeout object :param verify: (optional) Either a boolean, in which case it controls whether we verify the server's TLS certificate, or a string, in which case it must be a path to a CA bundle to use :param cert: (optional) Any user-provided SSL certificate to be trusted. :param proxies: (optional) The proxies dictionary to apply to the request. :rtype: requests.Response """ assert _is_prepared(request) try: conn = self.get_connection_with_tls_context( request, verify, proxies=proxies, cert=cert ) except LocationValueError as e: raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) self.add_headers( request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies, ) chunked = not (request.body is None or "Content-Length" in request.headers) if isinstance(timeout, tuple): try: connect, read = timeout resolved_timeout = TimeoutSauce(connect=connect, read=read) except ValueError: raise ValueError( f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, " f"or a single float to set both timeouts to the same value." ) elif isinstance(timeout, TimeoutSauce): resolved_timeout = timeout else: resolved_timeout = TimeoutSauce(connect=timeout, read=timeout) try: resp = conn.urlopen( method=request.method, url=url, body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str] headers=request.headers, # type: ignore[arg-type] # urllib3#3072 redirect=False, assert_same_host=False, preload_content=False, decode_content=False, retries=self.max_retries, timeout=resolved_timeout, chunked=chunked, ) except (ProtocolError, OSError) as err: raise ConnectionError(err, request=request) except MaxRetryError as e: if isinstance(e.reason, ConnectTimeoutError): # TODO: Remove this in 3.0.0: see #2811 if not isinstance(e.reason, NewConnectionError): raise ConnectTimeout(e, request=request) if isinstance(e.reason, ResponseError): raise RetryError(e, request=request) if isinstance(e.reason, _ProxyError): raise ProxyError(e, request=request) if isinstance(e.reason, _SSLError): # This branch is for urllib3 v1.22 and later. raise SSLError(e, request=request) > raise ConnectionError(e, request=request) E requests.exceptions.ConnectionError: HTTPSConnectionPool(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Max retries exceeded with url: /maas-api/v1/api-keys (Caused by NameResolutionError("HTTPSConnection(host='maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev', port=443): Failed to resolve 'maas.apps.64f26d0cc47cb9679a05.prod.konflux-ocp-ci.dev' ([Errno -2] Name or service not known)")) test/e2e/.venv/lib64/python3.11/site-packages/requests/adapters.py:729: ConnectionError