New upstream version 2.14.6

5 files changed, 86 insertions, 61 deletions

diff --git a/lib/ansible/galaxy/api.py b/lib/ansible/galaxy/api.py
index 8dea8049..0d519980 100644
--- a/lib/ansible/galaxy/api.py
+++ b/lib/ansible/galaxy/api.py
@@ -11,12 +11,15 @@ import functools
 import hashlib
 import json
 import os
+import socket
 import stat
 import tarfile
 import time
 import threading
 
-from urllib.error import HTTPError
+from http import HTTPStatus
+from http.client import BadStatusLine, IncompleteRead
+from urllib.error import HTTPError, URLError
 from urllib.parse import quote as urlquote, urlencode, urlparse, parse_qs, urljoin
 
 from ansible import constants as C
@@ -34,10 +37,11 @@ from ansible.utils.path import makedirs_safe
 display = Display()
 _CACHE_LOCK = threading.Lock()
 COLLECTION_PAGE_SIZE = 100
-RETRY_HTTP_ERROR_CODES = [  # TODO: Allow user-configuration
-    429,  # Too Many Requests
+RETRY_HTTP_ERROR_CODES = {  # TODO: Allow user-configuration
+    HTTPStatus.TOO_MANY_REQUESTS,
     520,  # Galaxy rate limit error code (Cloudflare unknown error)
-]
+    HTTPStatus.BAD_GATEWAY,  # Common error from galaxy that may represent any number of transient backend issues
+}
 
 
 def cache_lock(func):
@@ -48,11 +52,24 @@ def cache_lock(func):
     return wrapped
 
 
-def is_rate_limit_exception(exception):
+def should_retry_error(exception):
     # Note: cloud.redhat.com masks rate limit errors with 403 (Forbidden) error codes.
     # Since 403 could reflect the actual problem (such as an expired token), we should
     # not retry by default.
-    return isinstance(exception, GalaxyError) and exception.http_code in RETRY_HTTP_ERROR_CODES
+    if isinstance(exception, GalaxyError) and exception.http_code in RETRY_HTTP_ERROR_CODES:
+        return True
+
+    if isinstance(exception, AnsibleError) and (orig_exc := getattr(exception, 'orig_exc', None)):
+        # URLError is often a proxy for an underlying error, handle wrapped exceptions
+        if isinstance(orig_exc, URLError):
+            orig_exc = orig_exc.reason
+
+        # Handle common URL related errors such as TimeoutError, and BadStatusLine
+        # Note: socket.timeout is only required for Py3.9
+        if isinstance(orig_exc, (TimeoutError, BadStatusLine, IncompleteRead, socket.timeout)):
+            return True
+
+    return False
 
 
 def g_connect(versions):
@@ -327,7 +344,7 @@ class GalaxyAPI:
 
     @retry_with_delays_and_condition(
         backoff_iterator=generate_jittered_backoff(retries=6, delay_base=2, delay_threshold=40),
-        should_retry_error=is_rate_limit_exception
+        should_retry_error=should_retry_error
     )
     def _call_galaxy(self, url, args=None, headers=None, method=None, auth_required=False, error_context_msg=None,
                      cache=False, cache_key=None):
@@ -385,7 +402,10 @@ class GalaxyAPI:
         except HTTPError as e:
             raise GalaxyError(e, error_context_msg)
         except Exception as e:
-            raise AnsibleError("Unknown error when attempting to call Galaxy at '%s': %s" % (url, to_native(e)))
+            raise AnsibleError(
+                "Unknown error when attempting to call Galaxy at '%s': %s" % (url, to_native(e)),
+                orig_exc=e
+            )
 
         resp_data = to_text(resp.read(), errors='surrogate_or_strict')
         try:
@@ -906,8 +926,7 @@ class GalaxyAPI:
         try:
             signatures = data["signatures"]
         except KeyError:
-            # Noisy since this is used by the dep resolver, so require more verbosity than Galaxy calls
-            display.vvvvvv(f"Server {self.api_server} has not signed {namespace}.{name}:{version}")
+            display.vvvv(f"Server {self.api_server} has not signed {namespace}.{name}:{version}")
             return []
         else:
             return [signature_info["signature"] for signature_info in signatures]
diff --git a/lib/ansible/galaxy/collection/__init__.py b/lib/ansible/galaxy/collection/__init__.py
index 7a144c0d..23482665 100644
--- a/lib/ansible/galaxy/collection/__init__.py
+++ b/lib/ansible/galaxy/collection/__init__.py
@@ -767,6 +767,9 @@ def install_collections(
                     "Skipping signature verification."
                 )
 
+            if concrete_coll_pin.type == 'galaxy':
+                concrete_coll_pin = concrete_coll_pin.with_signatures_repopulated()
+
             try:
                 install(concrete_coll_pin, output_path, artifacts_manager)
             except AnsibleError as err:
@@ -911,7 +914,7 @@ def verify_collections(
                         # NOTE: If there are no Galaxy server signatures, only user-provided signature URLs,
                         # NOTE: those alone validate the MANIFEST.json and the remote collection is not downloaded.
                         # NOTE: The remote MANIFEST.json is only used in verification if there are no signatures.
-                        if not signatures and not collection.signature_sources:
+                        if artifacts_manager.keyring is None or not signatures:
                             api_proxy.get_collection_version_metadata(
                                 remote_collection,
                             )
diff --git a/lib/ansible/galaxy/collection/concrete_artifact_manager.py b/lib/ansible/galaxy/collection/concrete_artifact_manager.py
index 7c920b85..67d8e43f 100644
--- a/lib/ansible/galaxy/collection/concrete_artifact_manager.py
+++ b/lib/ansible/galaxy/collection/concrete_artifact_manager.py
@@ -27,9 +27,12 @@ if t.TYPE_CHECKING:
 
 from ansible.errors import AnsibleError
 from ansible.galaxy import get_collections_galaxy_meta_info
+from ansible.galaxy.api import should_retry_error
 from ansible.galaxy.dependency_resolution.dataclasses import _GALAXY_YAML
 from ansible.galaxy.user_agent import user_agent
 from ansible.module_utils._text import to_bytes, to_native, to_text
+from ansible.module_utils.api import retry_with_delays_and_condition
+from ansible.module_utils.api import generate_jittered_backoff
 from ansible.module_utils.common.process import get_bin_path
 from ansible.module_utils.common._collections_compat import MutableMapping
 from ansible.module_utils.common.yaml import yaml_load
@@ -160,17 +163,24 @@ class ConcreteArtifactsManager:
                 token=token,
             )  # type: bytes
         except URLError as err:
-            raise_from(
-                AnsibleError(
-                    'Failed to download collection tar '
-                    "from '{coll_src!s}': {download_err!s}".
-                    format(
-                        coll_src=to_native(collection.src),
-                        download_err=to_native(err),
-                    ),
+            raise AnsibleError(
+                'Failed to download collection tar '
+                "from '{coll_src!s}': {download_err!s}".
+                format(
+                    coll_src=to_native(collection.src),
+                    download_err=to_native(err),
                 ),
-                err,
-            )
+            ) from err
+        except Exception as err:
+            raise AnsibleError(
+                'Failed to download collection tar '
+                "from '{coll_src!s}' due to the following unforeseen error: "
+                '{download_err!s}'.
+                format(
+                    coll_src=to_native(collection.src),
+                    download_err=to_native(err),
+                ),
+            ) from err
         else:
             display.vvv(
                 "Collection '{coll!s}' obtained from "
@@ -460,6 +470,10 @@ def _extract_collection_from_git(repo_url, coll_ver, b_path):
 
 
 # FIXME: use random subdirs while preserving the file names
+@retry_with_delays_and_condition(
+    backoff_iterator=generate_jittered_backoff(retries=6, delay_base=2, delay_threshold=40),
+    should_retry_error=should_retry_error
+)
 def _download_file(url, b_path, expected_hash, validate_certs, token=None, timeout=60):
     # type: (str, bytes, t.Optional[str], bool, GalaxyToken, int) -> bytes
     # ^ NOTE: used in download and verify_collections ^
@@ -478,13 +492,16 @@ def _download_file(url, b_path, expected_hash, validate_certs, token=None, timeo
     display.display("Downloading %s to %s" % (url, to_text(b_tarball_dir)))
     # NOTE: Galaxy redirects downloads to S3 which rejects the request
     # NOTE: if an Authorization header is attached so don't redirect it
-    resp = open_url(
-        to_native(url, errors='surrogate_or_strict'),
-        validate_certs=validate_certs,
-        headers=None if token is None else token.headers(),
-        unredirected_headers=['Authorization'], http_agent=user_agent(),
-        timeout=timeout
-    )
+    try:
+        resp = open_url(
+            to_native(url, errors='surrogate_or_strict'),
+            validate_certs=validate_certs,
+            headers=None if token is None else token.headers(),
+            unredirected_headers=['Authorization'], http_agent=user_agent(),
+            timeout=timeout
+        )
+    except Exception as err:
+        raise AnsibleError(to_native(err), orig_exc=err)
 
     with open(b_file_path, 'wb') as download_file:  # type: t.BinaryIO
         actual_hash = _consume_file(resp, write_to=download_file)
diff --git a/lib/ansible/galaxy/dependency_resolution/dataclasses.py b/lib/ansible/galaxy/dependency_resolution/dataclasses.py
index 16fd6318..32acabdf 100644
--- a/lib/ansible/galaxy/dependency_resolution/dataclasses.py
+++ b/lib/ansible/galaxy/dependency_resolution/dataclasses.py
@@ -27,7 +27,7 @@ if t.TYPE_CHECKING:
     )
 
 
-from ansible.errors import AnsibleError
+from ansible.errors import AnsibleError, AnsibleAssertionError
 from ansible.galaxy.api import GalaxyAPI
 from ansible.module_utils._text import to_bytes, to_native, to_text
 from ansible.module_utils.common.arg_spec import ArgumentSpecValidator
@@ -166,6 +166,7 @@ def _is_concrete_artifact_pointer(tested_str):
 
 
 class _ComputedReqKindsMixin:
+    UNIQUE_ATTRS = ('fqcn', 'ver', 'src', 'type')
 
     def __init__(self, *args, **kwargs):
         if not self.may_have_offline_galaxy_info:
@@ -180,6 +181,12 @@ class _ComputedReqKindsMixin:
                 self.ver
             )
 
+    def __hash__(self):
+        return hash(tuple(getattr(self, attr) for attr in _ComputedReqKindsMixin.UNIQUE_ATTRS))
+
+    def __eq__(self, candidate):
+        return hash(self) == hash(candidate)
+
     @classmethod
     def from_dir_path_as_unknown(  # type: ignore[misc]
             cls,  # type: t.Type[Collection]
@@ -571,3 +578,13 @@ class Candidate(
 
     def __init__(self, *args, **kwargs):
         super(Candidate, self).__init__()
+
+    def with_signatures_repopulated(self):  # type: (Candidate) -> Candidate
+        """Populate a new Candidate instance with Galaxy signatures.
+        :raises AnsibleAssertionError: If the supplied candidate is not sourced from a Galaxy-like index.
+        """
+        if self.type != 'galaxy':
+            raise AnsibleAssertionError(f"Invalid collection type for {self!r}: unable to get signatures from a galaxy server.")
+
+        signatures = self.src.get_collection_signatures(self.namespace, self.name, self.ver)
+        return self.__class__(self.fqcn, self.ver, self.src, self.type, frozenset([*self.signatures, *signatures]))
diff --git a/lib/ansible/galaxy/dependency_resolution/providers.py b/lib/ansible/galaxy/dependency_resolution/providers.py
index 817a1eb2..6ad1de84 100644
--- a/lib/ansible/galaxy/dependency_resolution/providers.py
+++ b/lib/ansible/galaxy/dependency_resolution/providers.py
@@ -28,8 +28,6 @@ from ansible.galaxy.dependency_resolution.versioning import (
 from ansible.module_utils.six import string_types
 from ansible.utils.version import SemanticVersion, LooseVersion
 
-from collections.abc import Set
-
 try:
     from resolvelib import AbstractProvider
     from resolvelib import __version__ as resolvelib_version
@@ -46,34 +44,6 @@ RESOLVELIB_UPPERBOUND = SemanticVersion("0.9.0")
 RESOLVELIB_VERSION = SemanticVersion.from_loose_version(LooseVersion(resolvelib_version))
 
 
-class PinnedCandidateRequests(Set):
-    """Custom set class to store Candidate objects. Excludes the 'signatures' attribute when determining if a Candidate instance is in the set."""
-    CANDIDATE_ATTRS = ('fqcn', 'ver', 'src', 'type')
-
-    def __init__(self, candidates):
-        self._candidates = set(candidates)
-
-    def __iter__(self):
-        return iter(self._candidates)
-
-    def __contains__(self, value):
-        if not isinstance(value, Candidate):
-            raise ValueError(f"Expected a Candidate object but got {value!r}")
-        for candidate in self._candidates:
-            # Compare Candidate attributes excluding "signatures" since it is
-            # unrelated to whether or not a matching Candidate is user-requested.
-            # Candidate objects in the set are not expected to have signatures.
-            for attr in PinnedCandidateRequests.CANDIDATE_ATTRS:
-                if getattr(value, attr) != getattr(candidate, attr):
-                    break
-            else:
-                return True
-        return False
-
-    def __len__(self):
-        return len(self._candidates)
-
-
 class CollectionDependencyProviderBase(AbstractProvider):
     """Delegate providing a requirement interface for the resolver."""
 
@@ -117,7 +87,7 @@ class CollectionDependencyProviderBase(AbstractProvider):
             Requirement.from_requirement_dict,
             art_mgr=concrete_artifacts_manager,
         )
-        self._pinned_candidate_requests = PinnedCandidateRequests(
+        self._pinned_candidate_requests = set(
             # NOTE: User-provided signatures are supplemental, so signatures
             # NOTE: are not used to determine if a candidate is user-requested
             Candidate(req.fqcn, req.ver, req.src, req.type, None)
@@ -392,7 +362,6 @@ class CollectionDependencyProviderBase(AbstractProvider):
 
             if not unsatisfied:
                 if self._include_signatures:
-                    signatures = src_server.get_collection_signatures(first_req.namespace, first_req.name, version)
                     for extra_source in extra_signature_sources:
                         signatures.append(get_signature_from_source(extra_source))
                 latest_matches.append(