New upstream version 2.14.0

15 files changed, 363 insertions, 215 deletions

diff --git a/lib/ansible/galaxy/api.py b/lib/ansible/galaxy/api.py
index 627f71fa..8dea8049 100644
--- a/lib/ansible/galaxy/api.py
+++ b/lib/ansible/galaxy/api.py
@@ -269,8 +269,9 @@ class GalaxyAPI:
         self.timeout = timeout
         self._available_api_versions = available_api_versions or {}
         self._priority = priority
+        self._server_timeout = timeout
 
-        b_cache_dir = to_bytes(C.config.get_config_value('GALAXY_CACHE_DIR'), errors='surrogate_or_strict')
+        b_cache_dir = to_bytes(C.GALAXY_CACHE_DIR, errors='surrogate_or_strict')
         makedirs_safe(b_cache_dir, mode=0o700)
         self._b_cache_path = os.path.join(b_cache_dir, b'api.json')
 
@@ -380,7 +381,7 @@ class GalaxyAPI:
         try:
             display.vvvv("Calling Galaxy at %s" % url)
             resp = open_url(to_native(url), data=args, validate_certs=self.validate_certs, headers=headers,
-                            method=method, timeout=self.timeout, http_agent=user_agent(), follow_redirects='safe')
+                            method=method, timeout=self._server_timeout, http_agent=user_agent(), follow_redirects='safe')
         except HTTPError as e:
             raise GalaxyError(e, error_context_msg)
         except Exception as e:
@@ -438,7 +439,14 @@ class GalaxyAPI:
         """
         url = _urljoin(self.api_server, self.available_api_versions['v1'], "tokens") + '/'
         args = urlencode({"github_token": github_token})
-        resp = open_url(url, data=args, validate_certs=self.validate_certs, method="POST", http_agent=user_agent(), timeout=self.timeout)
+
+        try:
+            resp = open_url(url, data=args, validate_certs=self.validate_certs, method="POST", http_agent=user_agent(), timeout=self._server_timeout)
+        except HTTPError as e:
+            raise GalaxyError(e, 'Attempting to authenticate to galaxy')
+        except Exception as e:
+            raise AnsibleError('Unable to authenticate to galaxy: %s' % to_native(e), orig_exc=e)
+
         data = json.loads(to_text(resp.read(), errors='surrogate_or_strict'))
         return data
 
diff --git a/lib/ansible/galaxy/collection/__init__.py b/lib/ansible/galaxy/collection/__init__.py
index 9812bcad..7f04052e 100644
--- a/lib/ansible/galaxy/collection/__init__.py
+++ b/lib/ansible/galaxy/collection/__init__.py
@@ -25,6 +25,7 @@ import typing as t
 
 from collections import namedtuple
 from contextlib import contextmanager
+from dataclasses import dataclass, fields as dc_fields
 from hashlib import sha256
 from io import BytesIO
 from importlib.metadata import distribution
@@ -40,6 +41,14 @@ except ImportError:
 else:
     HAS_PACKAGING = True
 
+try:
+    from distlib.manifest import Manifest  # type: ignore[import]
+    from distlib import DistlibException  # type: ignore[import]
+except ImportError:
+    HAS_DISTLIB = False
+else:
+    HAS_DISTLIB = True
+
 if t.TYPE_CHECKING:
     from ansible.galaxy.collection.concrete_artifact_manager import (
         ConcreteArtifactsManager,
@@ -112,12 +121,15 @@ from ansible.galaxy.dependency_resolution.dataclasses import (
     Candidate, Requirement, _is_installed_collection_dir,
 )
 from ansible.galaxy.dependency_resolution.versioning import meets_requirements
+from ansible.plugins.loader import get_all_plugin_loaders
 from ansible.module_utils.six import raise_from
 from ansible.module_utils._text import to_bytes, to_native, to_text
+from ansible.module_utils.common.collections import is_sequence
 from ansible.module_utils.common.yaml import yaml_dump
 from ansible.utils.collection_loader import AnsibleCollectionRef
 from ansible.utils.display import Display
 from ansible.utils.hashing import secure_hash, secure_hash_s
+from ansible.utils.sentinel import Sentinel
 
 
 display = Display()
@@ -130,6 +142,20 @@ ModifiedContent = namedtuple('ModifiedContent', ['filename', 'expected', 'instal
 SIGNATURE_COUNT_RE = r"^(?P<strict>\+)?(?:(?P<count>\d+)|(?P<all>all))$"
 
 
+@dataclass
+class ManifestControl:
+    directives: list[str] = None
+    omit_default_directives: bool = False
+
+    def __post_init__(self):
+        # Allow a dict representing this dataclass to be splatted directly.
+        # Requires attrs to have a default value, so anything with a default
+        # of None is swapped for its, potentially mutable, default
+        for field in dc_fields(self):
+            if getattr(self, field.name) is None:
+                super().__setattr__(field.name, field.type())
+
+
 class CollectionSignatureError(Exception):
     def __init__(self, reasons=None, stdout=None, rc=None, ignore=False):
         self.reasons = reasons
@@ -177,10 +203,8 @@ class CollectionVerifyResult:
         self.success = True  # type: bool
 
 
-def verify_local_collection(
-        local_collection, remote_collection,
-        artifacts_manager,
-):  # type: (Candidate, t.Optional[Candidate], ConcreteArtifactsManager) -> CollectionVerifyResult
+def verify_local_collection(local_collection, remote_collection, artifacts_manager):
+    # type: (Candidate, t.Optional[Candidate], ConcreteArtifactsManager) -> CollectionVerifyResult
     """Verify integrity of the locally installed collection.
 
     :param local_collection: Collection being checked.
@@ -190,9 +214,7 @@ def verify_local_collection(
     """
     result = CollectionVerifyResult(local_collection.fqcn)
 
-    b_collection_path = to_bytes(
-        local_collection.src, errors='surrogate_or_strict',
-    )
+    b_collection_path = to_bytes(local_collection.src, errors='surrogate_or_strict')
 
     display.display("Verifying '{coll!s}'.".format(coll=local_collection))
     display.display(
@@ -456,6 +478,7 @@ def build_collection(u_collection_path, u_output_path, force):
         collection_meta['namespace'],  # type: ignore[arg-type]
         collection_meta['name'],  # type: ignore[arg-type]
         collection_meta['build_ignore'],  # type: ignore[arg-type]
+        collection_meta['manifest'],  # type: ignore[arg-type]
     )
 
     artifact_tarball_file_name = '{ns!s}-{name!s}-{ver!s}.tar.gz'.format(
@@ -509,6 +532,7 @@ def download_collections(
             upgrade=False,
             # Avoid overhead getting signatures since they are not currently applicable to downloaded collections
             include_signatures=False,
+            offline=False,
         )
 
     b_output_path = to_bytes(output_path, errors='surrogate_or_strict')
@@ -634,6 +658,7 @@ def install_collections(
         allow_pre_release,  # type: bool
         artifacts_manager,  # type: ConcreteArtifactsManager
         disable_gpg_verify,  # type: bool
+        offline,  # type: bool
 ):  # type: (...) -> None
     """Install Ansible collections to the path specified.
 
@@ -711,6 +736,7 @@ def install_collections(
             allow_pre_release=allow_pre_release,
             upgrade=upgrade,
             include_signatures=not disable_gpg_verify,
+            offline=offline,
         )
 
     keyring_exists = artifacts_manager.keyring is not None
@@ -903,10 +929,7 @@ def verify_collections(
                             )
                         raise
 
-                result = verify_local_collection(
-                    local_collection, remote_collection,
-                    artifacts_manager,
-                )
+                result = verify_local_collection(local_collection, remote_collection, artifacts_manager)
 
                 results.append(result)
 
@@ -1014,7 +1037,146 @@ def _verify_file_hash(b_path, filename, expected_hash, error_queue):
         error_queue.append(ModifiedContent(filename=filename, expected=expected_hash, installed=actual_hash))
 
 
-def _build_files_manifest(b_collection_path, namespace, name, ignore_patterns):
+def _make_manifest():
+    return {
+        'files': [
+            {
+                'name': '.',
+                'ftype': 'dir',
+                'chksum_type': None,
+                'chksum_sha256': None,
+                'format': MANIFEST_FORMAT,
+            },
+        ],
+        'format': MANIFEST_FORMAT,
+    }
+
+
+def _make_entry(name, ftype, chksum_type='sha256', chksum=None):
+    return {
+        'name': name,
+        'ftype': ftype,
+        'chksum_type': chksum_type if chksum else None,
+        f'chksum_{chksum_type}': chksum,
+        'format': MANIFEST_FORMAT
+    }
+
+
+def _build_files_manifest(b_collection_path, namespace, name, ignore_patterns, manifest_control):
+    # type: (bytes, str, str, list[str], dict[str, t.Any]) -> FilesManifestType
+    if ignore_patterns and manifest_control is not Sentinel:
+        raise AnsibleError('"build_ignore" and "manifest" are mutually exclusive')
+
+    if manifest_control is not Sentinel:
+        return _build_files_manifest_distlib(
+            b_collection_path,
+            namespace,
+            name,
+            manifest_control,
+        )
+
+    return _build_files_manifest_walk(b_collection_path, namespace, name, ignore_patterns)
+
+
+def _build_files_manifest_distlib(b_collection_path, namespace, name, manifest_control):
+    # type: (bytes, str, str, dict[str, t.Any]) -> FilesManifestType
+
+    if not HAS_DISTLIB:
+        raise AnsibleError('Use of "manifest" requires the python "distlib" library')
+
+    if manifest_control is None:
+        manifest_control = {}
+
+    try:
+        control = ManifestControl(**manifest_control)
+    except TypeError as ex:
+        raise AnsibleError(f'Invalid "manifest" provided: {ex}')
+
+    if not is_sequence(control.directives):
+        raise AnsibleError(f'"manifest.directives" must be a list, got: {control.directives.__class__.__name__}')
+
+    if not isinstance(control.omit_default_directives, bool):
+        raise AnsibleError(
+            '"manifest.omit_default_directives" is expected to be a boolean, got: '
+            f'{control.omit_default_directives.__class__.__name__}'
+        )
+
+    if control.omit_default_directives and not control.directives:
+        raise AnsibleError(
+            '"manifest.omit_default_directives" was set to True, but no directives were defined '
+            'in "manifest.directives". This would produce an empty collection artifact.'
+        )
+
+    directives = []
+    if control.omit_default_directives:
+        directives.extend(control.directives)
+    else:
+        directives.extend([
+            'include meta/*.yml',
+            'include *.txt *.md *.rst COPYING LICENSE',
+            'recursive-include tests **',
+            'recursive-include docs **.rst **.yml **.yaml **.json **.j2 **.txt',
+            'recursive-include roles **.yml **.yaml **.json **.j2',
+            'recursive-include playbooks **.yml **.yaml **.json',
+            'recursive-include changelogs **.yml **.yaml',
+            'recursive-include plugins */**.py',
+        ])
+
+        plugins = set(l.package.split('.')[-1] for d, l in get_all_plugin_loaders())
+        for plugin in sorted(plugins):
+            if plugin in ('modules', 'module_utils'):
+                continue
+            elif plugin in C.DOCUMENTABLE_PLUGINS:
+                directives.append(
+                    f'recursive-include plugins/{plugin} **.yml **.yaml'
+                )
+
+        directives.extend([
+            'recursive-include plugins/modules **.ps1 **.yml **.yaml',
+            'recursive-include plugins/module_utils **.ps1 **.psm1 **.cs',
+        ])
+
+        directives.extend(control.directives)
+
+        directives.extend([
+            f'exclude galaxy.yml galaxy.yaml MANIFEST.json FILES.json {namespace}-{name}-*.tar.gz',
+            'recursive-exclude tests/output **',
+            'global-exclude /.* /__pycache__',
+        ])
+
+    display.vvv('Manifest Directives:')
+    display.vvv(textwrap.indent('\n'.join(directives), '    '))
+
+    u_collection_path = to_text(b_collection_path, errors='surrogate_or_strict')
+    m = Manifest(u_collection_path)
+    for directive in directives:
+        try:
+            m.process_directive(directive)
+        except DistlibException as e:
+            raise AnsibleError(f'Invalid manifest directive: {e}')
+        except Exception as e:
+            raise AnsibleError(f'Unknown error processing manifest directive: {e}')
+
+    manifest = _make_manifest()
+
+    for abs_path in m.sorted(wantdirs=True):
+        rel_path = os.path.relpath(abs_path, u_collection_path)
+        if os.path.isdir(abs_path):
+            manifest_entry = _make_entry(rel_path, 'dir')
+        else:
+            manifest_entry = _make_entry(
+                rel_path,
+                'file',
+                chksum_type='sha256',
+                chksum=secure_hash(abs_path, hash_func=sha256)
+            )
+
+        manifest['files'].append(manifest_entry)
+
+    return manifest
+
+
+def _build_files_manifest_walk(b_collection_path, namespace, name, ignore_patterns):
     # type: (bytes, str, str, list[str]) -> FilesManifestType
     # We always ignore .pyc and .retry files as well as some well known version control directories. The ignore
     # patterns can be extended by the build_ignore key in galaxy.yml
@@ -1032,25 +1194,7 @@ def _build_files_manifest(b_collection_path, namespace, name, ignore_patterns):
     b_ignore_patterns += [to_bytes(p) for p in ignore_patterns]
     b_ignore_dirs = frozenset([b'CVS', b'.bzr', b'.hg', b'.git', b'.svn', b'__pycache__', b'.tox'])
 
-    entry_template = {
-        'name': None,
-        'ftype': None,
-        'chksum_type': None,
-        'chksum_sha256': None,
-        'format': MANIFEST_FORMAT
-    }
-    manifest = {
-        'files': [
-            {
-                'name': '.',
-                'ftype': 'dir',
-                'chksum_type': None,
-                'chksum_sha256': None,
-                'format': MANIFEST_FORMAT,
-            },
-        ],
-        'format': MANIFEST_FORMAT,
-    }  # type: FilesManifestType
+    manifest = _make_manifest()
 
     def _walk(b_path, b_top_level_dir):
         for b_item in os.listdir(b_path):
@@ -1073,11 +1217,7 @@ def _build_files_manifest(b_collection_path, namespace, name, ignore_patterns):
                                         % to_text(b_abs_path))
                         continue
 
-                manifest_entry = entry_template.copy()
-                manifest_entry['name'] = rel_path
-                manifest_entry['ftype'] = 'dir'
-
-                manifest['files'].append(manifest_entry)
+                manifest['files'].append(_make_entry(rel_path, 'dir'))
 
                 if not os.path.islink(b_abs_path):
                     _walk(b_abs_path, b_top_level_dir)
@@ -1088,13 +1228,14 @@ def _build_files_manifest(b_collection_path, namespace, name, ignore_patterns):
 
                 # Handling of file symlinks occur in _build_collection_tar, the manifest for a symlink is the same for
                 # a normal file.
-                manifest_entry = entry_template.copy()
-                manifest_entry['name'] = rel_path
-                manifest_entry['ftype'] = 'file'
-                manifest_entry['chksum_type'] = 'sha256'
-                manifest_entry['chksum_sha256'] = secure_hash(b_abs_path, hash_func=sha256)
-
-                manifest['files'].append(manifest_entry)
+                manifest['files'].append(
+                    _make_entry(
+                        rel_path,
+                        'file',
+                        chksum_type='sha256',
+                        chksum=secure_hash(b_abs_path, hash_func=sha256)
+                    )
+                )
 
     _walk(b_collection_path, b_collection_path)
 
@@ -1267,10 +1408,7 @@ def find_existing_collections(path, artifacts_manager):
                 continue
 
             try:
-                req = Candidate.from_dir_path_as_unknown(
-                    b_collection_path,
-                    artifacts_manager,
-                )
+                req = Candidate.from_dir_path_as_unknown(b_collection_path, artifacts_manager)
             except ValueError as val_err:
                 raise_from(AnsibleError(val_err), val_err)
 
@@ -1411,11 +1549,7 @@ def install_artifact(b_coll_targz_path, b_collection_path, b_temp_path, signatur
         raise
 
 
-def install_src(
-        collection,
-        b_collection_path, b_collection_output_path,
-        artifacts_manager,
-):
+def install_src(collection, b_collection_path, b_collection_output_path, artifacts_manager):
     r"""Install the collection from source control into given dir.
 
     Generates the Ansible collection artifact data from a galaxy.yml and
@@ -1441,6 +1575,7 @@ def install_src(
         b_collection_path,
         collection_meta['namespace'], collection_meta['name'],
         collection_meta['build_ignore'],
+        collection_meta['manifest'],
     )
 
     collection_output_path = _build_collection_dir(
@@ -1600,16 +1735,20 @@ def _resolve_depenency_map(
         allow_pre_release,  # type: bool
         upgrade,  # type: bool
         include_signatures,  # type: bool
+        offline,  # type: bool
 ):  # type: (...) -> dict[str, Candidate]
     """Return the resolved dependency map."""
     if not HAS_RESOLVELIB:
         raise AnsibleError("Failed to import resolvelib, check that a supported version is installed")
     if not HAS_PACKAGING:
         raise AnsibleError("Failed to import packaging, check that a supported version is installed")
+
+    req = None
+
     try:
         dist = distribution('ansible-core')
     except Exception:
-        req = None
+        pass
     else:
         req = next((rr for r in (dist.requires or []) if (rr := PkgReq(r)).name == 'resolvelib'), None)
     finally:
@@ -1632,6 +1771,7 @@ def _resolve_depenency_map(
         with_pre_releases=allow_pre_release,
         upgrade=upgrade,
         include_signatures=include_signatures,
+        offline=offline,
     )
     try:
         return collection_dep_resolver.resolve(
diff --git a/lib/ansible/galaxy/collection/concrete_artifact_manager.py b/lib/ansible/galaxy/collection/concrete_artifact_manager.py
index 4115aeed..7c920b85 100644
--- a/lib/ansible/galaxy/collection/concrete_artifact_manager.py
+++ b/lib/ansible/galaxy/collection/concrete_artifact_manager.py
@@ -36,6 +36,7 @@ from ansible.module_utils.common.yaml import yaml_load
 from ansible.module_utils.six import raise_from
 from ansible.module_utils.urls import open_url
 from ansible.utils.display import Display
+from ansible.utils.sentinel import Sentinel
 
 import yaml
 
@@ -64,7 +65,7 @@ class ConcreteArtifactsManager:
         self._validate_certs = validate_certs  # type: bool
         self._artifact_cache = {}  # type: dict[bytes, bytes]
         self._galaxy_artifact_cache = {}  # type: dict[Candidate | Requirement, bytes]
-        self._artifact_meta_cache = {}  # type: dict[bytes, dict[str, str | list[str] | dict[str, str] | None]]
+        self._artifact_meta_cache = {}  # type: dict[bytes, dict[str, str | list[str] | dict[str, str] | None | t.Type[Sentinel]]]
         self._galaxy_collection_cache = {}  # type: dict[Candidate | Requirement, tuple[str, str, GalaxyToken]]
         self._galaxy_collection_origin_cache = {}  # type: dict[Candidate, tuple[str, list[dict[str, str]]]]
         self._b_working_directory = b_working_directory  # type: bytes
@@ -218,7 +219,7 @@ class ConcreteArtifactsManager:
                     validate_certs=self._validate_certs,
                     timeout=self.timeout
                 )
-            except URLError as err:
+            except Exception as err:
                 raise_from(
                     AnsibleError(
                         'Failed to download collection tar '
@@ -280,10 +281,13 @@ class ConcreteArtifactsManager:
     def get_direct_collection_dependencies(self, collection):
         # type: (t.Union[Candidate, Requirement]) -> dict[str, str]
         """Extract deps from the given on-disk collection artifact."""
-        return self.get_direct_collection_meta(collection)['dependencies']  # type: ignore[return-value]
+        collection_dependencies = self.get_direct_collection_meta(collection)['dependencies']
+        if collection_dependencies is None:
+            collection_dependencies = {}
+        return collection_dependencies  # type: ignore[return-value]
 
     def get_direct_collection_meta(self, collection):
-        # type: (t.Union[Candidate, Requirement]) -> dict[str, t.Union[str, dict[str, str], list[str], None]]
+        # type: (t.Union[Candidate, Requirement]) -> dict[str, t.Union[str, dict[str, str], list[str], None, t.Type[Sentinel]]]
         """Extract meta from the given on-disk collection artifact."""
         try:  # FIXME: use unique collection identifier as a cache key?
             return self._artifact_meta_cache[collection.src]
@@ -513,11 +517,11 @@ def _consume_file(read_from, write_to=None):
 
 
 def _normalize_galaxy_yml_manifest(
-        galaxy_yml,  # type: dict[str, t.Union[str, list[str], dict[str, str], None]]
+        galaxy_yml,  # type: dict[str, t.Union[str, list[str], dict[str, str], None, t.Type[Sentinel]]]
         b_galaxy_yml_path,  # type: bytes
         require_build_metadata=True,  # type: bool
 ):
-    # type: (...) -> dict[str, t.Union[str, list[str], dict[str, str], None]]
+    # type: (...) -> dict[str, t.Union[str, list[str], dict[str, str], None, t.Type[Sentinel]]]
     galaxy_yml_schema = (
         get_collections_galaxy_meta_info()
     )  # type: list[dict[str, t.Any]]  # FIXME: <--
@@ -527,6 +531,7 @@ def _normalize_galaxy_yml_manifest(
     string_keys = set()  # type: set[str]
     list_keys = set()  # type: set[str]
     dict_keys = set()  # type: set[str]
+    sentinel_keys = set()  # type: set[str]
 
     for info in galaxy_yml_schema:
         if info.get('required', False):
@@ -536,10 +541,11 @@ def _normalize_galaxy_yml_manifest(
             'str': string_keys,
             'list': list_keys,
             'dict': dict_keys,
+            'sentinel': sentinel_keys,
         }[info.get('type', 'str')]
         key_list_type.add(info['key'])
 
-    all_keys = frozenset(list(mandatory_keys) + list(string_keys) + list(list_keys) + list(dict_keys))
+    all_keys = frozenset(mandatory_keys | string_keys | list_keys | dict_keys | sentinel_keys)
 
     set_keys = set(galaxy_yml.keys())
     missing_keys = mandatory_keys.difference(set_keys)
@@ -575,6 +581,10 @@ def _normalize_galaxy_yml_manifest(
         if optional_dict not in galaxy_yml:
             galaxy_yml[optional_dict] = {}
 
+    for optional_sentinel in sentinel_keys:
+        if optional_sentinel not in galaxy_yml:
+            galaxy_yml[optional_sentinel] = Sentinel
+
     # NOTE: `version: null` is only allowed for `galaxy.yml`
     # NOTE: and not `MANIFEST.json`. The use-case for it is collections
     # NOTE: that generate the version from Git before building a
@@ -588,7 +598,7 @@ def _normalize_galaxy_yml_manifest(
 def _get_meta_from_dir(
         b_path,  # type: bytes
         require_build_metadata=True,  # type: bool
-):  # type: (...) -> dict[str, t.Union[str, list[str], dict[str, str], None]]
+):  # type: (...) -> dict[str, t.Union[str, list[str], dict[str, str], None, t.Type[Sentinel]]]
     try:
         return _get_meta_from_installed_dir(b_path)
     except LookupError:
@@ -598,7 +608,7 @@ def _get_meta_from_dir(
 def _get_meta_from_src_dir(
         b_path,  # type: bytes
         require_build_metadata=True,  # type: bool
-):  # type: (...) -> dict[str, t.Union[str, list[str], dict[str, str], None]]
+):  # type: (...) -> dict[str, t.Union[str, list[str], dict[str, str], None, t.Type[Sentinel]]]
     galaxy_yml = os.path.join(b_path, _GALAXY_YAML)
     if not os.path.isfile(galaxy_yml):
         raise LookupError(
@@ -667,7 +677,7 @@ def _get_json_from_installed_dir(
 
 def _get_meta_from_installed_dir(
         b_path,  # type: bytes
-):  # type: (...) -> dict[str, t.Union[str, list[str], dict[str, str], None]]
+):  # type: (...) -> dict[str, t.Union[str, list[str], dict[str, str], None, t.Type[Sentinel]]]
     manifest = _get_json_from_installed_dir(b_path, MANIFEST_FILENAME)
     collection_info = manifest['collection_info']
 
@@ -688,7 +698,7 @@ def _get_meta_from_installed_dir(
 
 def _get_meta_from_tar(
         b_path,  # type: bytes
-):  # type: (...) -> dict[str, t.Union[str, list[str], dict[str, str], None]]
+):  # type: (...) -> dict[str, t.Union[str, list[str], dict[str, str], None, t.Type[Sentinel]]]
     if not tarfile.is_tarfile(b_path):
         raise AnsibleError(
             "Collection artifact at '{path!s}' is not a valid tar file.".
diff --git a/lib/ansible/galaxy/collection/galaxy_api_proxy.py b/lib/ansible/galaxy/collection/galaxy_api_proxy.py
index ae104c4c..51e0c9f5 100644
--- a/lib/ansible/galaxy/collection/galaxy_api_proxy.py
+++ b/lib/ansible/galaxy/collection/galaxy_api_proxy.py
@@ -28,11 +28,20 @@ display = Display()
 class MultiGalaxyAPIProxy:
     """A proxy that abstracts talking to multiple Galaxy instances."""
 
-    def __init__(self, apis, concrete_artifacts_manager):
-        # type: (t.Iterable[GalaxyAPI], ConcreteArtifactsManager) -> None
+    def __init__(self, apis, concrete_artifacts_manager, offline=False):
+        # type: (t.Iterable[GalaxyAPI], ConcreteArtifactsManager, bool) -> None
         """Initialize the target APIs list."""
         self._apis = apis
         self._concrete_art_mgr = concrete_artifacts_manager
+        self._offline = offline  # Prevent all GalaxyAPI calls
+
+    @property
+    def is_offline_mode_requested(self):
+        return self._offline
+
+    def _assert_that_offline_mode_is_not_requested(self):  # type: () -> None
+        if self.is_offline_mode_requested:
+            raise NotImplementedError("The calling code is not supposed to be invoked in 'offline' mode.")
 
     def _get_collection_versions(self, requirement):
         # type: (Requirement) -> t.Iterator[tuple[GalaxyAPI, str]]
@@ -41,6 +50,9 @@ class MultiGalaxyAPIProxy:
         Yield api, version pairs for all APIs,
         and reraise the last error if no valid API was found.
         """
+        if self._offline:
+            return []
+
         found_api = False
         last_error = None  # type: Exception | None
 
@@ -102,6 +114,7 @@ class MultiGalaxyAPIProxy:
     def get_collection_version_metadata(self, collection_candidate):
         # type: (Candidate) -> CollectionVersionMetadata
         """Retrieve collection metadata of a given candidate."""
+        self._assert_that_offline_mode_is_not_requested()
 
         api_lookup_order = (
             (collection_candidate.src, )
@@ -167,6 +180,7 @@ class MultiGalaxyAPIProxy:
 
     def get_signatures(self, collection_candidate):
         # type: (Candidate) -> list[str]
+        self._assert_that_offline_mode_is_not_requested()
         namespace = collection_candidate.namespace
         name = collection_candidate.name
         version = collection_candidate.ver
diff --git a/lib/ansible/galaxy/data/apb/.travis.yml b/lib/ansible/galaxy/data/apb/.travis.yml
deleted file mode 100644
index 44c0ba40..00000000
--- a/lib/ansible/galaxy/data/apb/.travis.yml
+++ /dev/null
@@ -1,25 +0,0 @@
----
-services: docker
-sudo: required
-language: python
-python:
-  - '2.7'
-
-env:
-  - OPENSHIFT_VERSION=v3.9.0
-  - KUBERNETES_VERSION=v1.9.0
-
-script:
-  # Configure test values
-  - export apb_name=APB_NAME
-
-  # Download test shim.
-  - wget -O ${PWD}/apb-test.sh https://raw.githubusercontent.com/ansibleplaybookbundle/apb-test-shim/master/apb-test.sh
-  - chmod +x ${PWD}/apb-test.sh
-
-  # Run tests.
-  - ${PWD}/apb-test.sh
-
-# Uncomment to allow travis to notify galaxy
-# notifications:
-#  webhooks: https://galaxy.ansible.com/api/v1/notifications/
diff --git a/lib/ansible/galaxy/data/collections_galaxy_meta.yml b/lib/ansible/galaxy/data/collections_galaxy_meta.yml
index 75137234..5c4472cd 100644
--- a/lib/ansible/galaxy/data/collections_galaxy_meta.yml
+++ b/lib/ansible/galaxy/data/collections_galaxy_meta.yml
@@ -106,5 +106,15 @@
   - This uses C(fnmatch) to match the files or directories.
   - Some directories and files like C(galaxy.yml), C(*.pyc), C(*.retry), and
     C(.git) are always filtered.
+  - Mutually exclusive with C(manifest)
   type: list
   version_added: '2.10'
+
+- key: manifest
+  description:
+  - A dict controlling use of manifest directives used in building the collection artifact.
+  - The key C(directives) is a list of MANIFEST.in style L(directives,https://packaging.python.org/en/latest/guides/using-manifest-in/#manifest-in-commands)
+  - The key C(omit_default_directives) is a boolean that controls whether the default directives are used
+  - Mutually exclusive with C(build_ignore)
+  type: sentinel
+  version_added: '2.14'
diff --git a/lib/ansible/galaxy/data/container/.travis.yml b/lib/ansible/galaxy/data/container/.travis.yml
deleted file mode 100644
index a3370b7d..00000000
--- a/lib/ansible/galaxy/data/container/.travis.yml
+++ /dev/null
@@ -1,45 +0,0 @@
-language: python
-dist: trusty
-sudo: required
-
-services:
-  - docker
-
-before_install:
-  - sudo apt-add-repository 'deb http://archive.ubuntu.com/ubuntu trusty-backports universe'
-  - sudo apt-get update -qq
-  - sudo apt-get install -y -o Dpkg::Options::="--force-confold" --force-yes docker-engine
-
-install:
-  # Install the latest Ansible Container and Ansible
-  - pip install git+https://github.com/ansible/ansible-container.git
-  - pip install ansible
-
-script:
-  # Make sure docker is functioning
-  - docker version
-  - docker-compose version
-  - docker info
-
-  # Create an Ansible Container project
-  - mkdir -p tests
-  - cd tests
-  - ansible-container init
-
-  # Install the role into the project
-  - echo "Installing and testing git+https://github.com/${TRAVIS_REPO_SLUG},${TRAVIS_COMMIT}"
-  - ansible-container install git+https://github.com/${TRAVIS_REPO_SLUG},${TRAVIS_COMMIT}
-
-  # Build the service image
-  - ansible-container build
-
-  # Start the service
-  - ansible-container run -d
-  - docker ps
-
-  # Run tests
-  - ansible-playbook test.yml
-
-notifications:
-  email: false
-  webhooks: https://galaxy.ansible.com/api/v1/notifications/
diff --git a/lib/ansible/galaxy/data/default/collection/galaxy.yml.j2 b/lib/ansible/galaxy/data/default/collection/galaxy.yml.j2
index a95008fc..7821491b 100644
--- a/lib/ansible/galaxy/data/default/collection/galaxy.yml.j2
+++ b/lib/ansible/galaxy/data/default/collection/galaxy.yml.j2
@@ -7,5 +7,10 @@
 ### OPTIONAL but strongly recommended
 {% for option in optional_config %}
 {{ option.description | comment_ify }}
+{% if option.key == 'manifest' %}
+{{ {option.key: option.value} | to_nice_yaml | comment_ify }}
+
+{% else %}
 {{ {option.key: option.value} | to_nice_yaml }}
+{% endif %}
 {% endfor %}
diff --git a/lib/ansible/galaxy/data/default/collection/meta/runtime.yml b/lib/ansible/galaxy/data/default/collection/meta/runtime.yml
new file mode 100644
index 00000000..20f709ed
--- /dev/null
+++ b/lib/ansible/galaxy/data/default/collection/meta/runtime.yml
@@ -0,0 +1,52 @@
+---
+# Collections must specify a minimum required ansible version to upload
+# to galaxy
+# requires_ansible: '>=2.9.10'
+
+# Content that Ansible needs to load from another location or that has
+# been deprecated/removed
+# plugin_routing:
+#   action:
+#     redirected_plugin_name:
+#       redirect: ns.col.new_location
+#     deprecated_plugin_name:
+#       deprecation:
+#         removal_version: "4.0.0"
+#         warning_text: |
+#           See the porting guide on how to update your playbook to
+#           use ns.col.another_plugin instead.
+#     removed_plugin_name:
+#       tombstone:
+#         removal_version: "2.0.0"
+#         warning_text: |
+#           See the porting guide on how to update your playbook to
+#           use ns.col.another_plugin instead.
+#   become:
+#   cache:
+#   callback:
+#   cliconf:
+#   connection:
+#   doc_fragments:
+#   filter:
+#   httpapi:
+#   inventory:
+#   lookup:
+#   module_utils:
+#   modules:
+#   netconf:
+#   shell:
+#   strategy:
+#   terminal:
+#   test:
+#   vars:
+
+# Python import statements that Ansible needs to load from another location
+# import_redirection:
+#   ansible_collections.ns.col.plugins.module_utils.old_location:
+#     redirect: ansible_collections.ns.col.plugins.module_utils.new_location
+
+# Groups of actions/modules that take a common set of options
+# action_groups:
+#   group_name:
+#     - module1
+#     - module2
diff --git a/lib/ansible/galaxy/data/default/role/.travis.yml b/lib/ansible/galaxy/data/default/role/.travis.yml
deleted file mode 100644
index 36bbf620..00000000
--- a/lib/ansible/galaxy/data/default/role/.travis.yml
+++ /dev/null
@@ -1,29 +0,0 @@
----
-language: python
-python: "2.7"
-
-# Use the new container infrastructure
-sudo: false
-
-# Install ansible
-addons:
-  apt:
-    packages:
-    - python-pip
-
-install:
-  # Install ansible
-  - pip install ansible
-
-  # Check ansible version
-  - ansible --version
-
-  # Create ansible.cfg with correct roles_path
-  - printf '[defaults]\nroles_path=../' >ansible.cfg
-
-script:
-  # Basic role syntax check
-  - ansible-playbook tests/test.yml -i tests/inventory --syntax-check
-
-notifications:
-  webhooks: https://galaxy.ansible.com/api/v1/notifications/
\ No newline at end of file
diff --git a/lib/ansible/galaxy/data/network/.travis.yml b/lib/ansible/galaxy/data/network/.travis.yml
deleted file mode 100644
index 36bbf620..00000000
--- a/lib/ansible/galaxy/data/network/.travis.yml
+++ /dev/null
@@ -1,29 +0,0 @@
----
-language: python
-python: "2.7"
-
-# Use the new container infrastructure
-sudo: false
-
-# Install ansible
-addons:
-  apt:
-    packages:
-    - python-pip
-
-install:
-  # Install ansible
-  - pip install ansible
-
-  # Check ansible version
-  - ansible --version
-
-  # Create ansible.cfg with correct roles_path
-  - printf '[defaults]\nroles_path=../' >ansible.cfg
-
-script:
-  # Basic role syntax check
-  - ansible-playbook tests/test.yml -i tests/inventory --syntax-check
-
-notifications:
-  webhooks: https://galaxy.ansible.com/api/v1/notifications/
\ No newline at end of file
diff --git a/lib/ansible/galaxy/dependency_resolution/__init__.py b/lib/ansible/galaxy/dependency_resolution/__init__.py
index 2de16737..cfde7df0 100644
--- a/lib/ansible/galaxy/dependency_resolution/__init__.py
+++ b/lib/ansible/galaxy/dependency_resolution/__init__.py
@@ -33,6 +33,7 @@ def build_collection_dependency_resolver(
         with_pre_releases=False,  # type: bool
         upgrade=False,  # type: bool
         include_signatures=True,  # type: bool
+        offline=False,  # type: bool
 ):  # type: (...) -> CollectionDependencyResolver
     """Return a collection dependency resolver.
 
@@ -41,7 +42,7 @@ def build_collection_dependency_resolver(
     """
     return CollectionDependencyResolver(
         CollectionDependencyProvider(
-            apis=MultiGalaxyAPIProxy(galaxy_apis, concrete_artifacts_manager),
+            apis=MultiGalaxyAPIProxy(galaxy_apis, concrete_artifacts_manager, offline=offline),
             concrete_artifacts_manager=concrete_artifacts_manager,
             user_requirements=user_requirements,
             preferred_candidates=preferred_candidates,
diff --git a/lib/ansible/galaxy/dependency_resolution/dataclasses.py b/lib/ansible/galaxy/dependency_resolution/dataclasses.py
index 2ac68925..16fd6318 100644
--- a/lib/ansible/galaxy/dependency_resolution/dataclasses.py
+++ b/lib/ansible/galaxy/dependency_resolution/dataclasses.py
@@ -193,7 +193,7 @@ class _ComputedReqKindsMixin:
         falls back to guessing the FQCN based on the directory path and
         sets the version to "*".
 
-        It raises a ValueError immediatelly if the input is not an
+        It raises a ValueError immediately if the input is not an
         existing directory path.
         """
         if not os.path.isdir(dir_path):
@@ -280,7 +280,7 @@ class _ComputedReqKindsMixin:
         return cls.from_requirement_dict(req, artifacts_manager)
 
     @classmethod
-    def from_requirement_dict(cls, collection_req, art_mgr):
+    def from_requirement_dict(cls, collection_req, art_mgr, validate_signature_options=True):
         req_name = collection_req.get('name', None)
         req_version = collection_req.get('version', '*')
         req_type = collection_req.get('type')
@@ -288,7 +288,7 @@ class _ComputedReqKindsMixin:
         req_source = collection_req.get('source', None)
         req_signature_sources = collection_req.get('signatures', None)
         if req_signature_sources is not None:
-            if art_mgr.keyring is None:
+            if validate_signature_options and art_mgr.keyring is None:
                 raise AnsibleError(
                     f"Signatures were provided to verify {req_name} but no keyring was configured."
                 )
diff --git a/lib/ansible/galaxy/dependency_resolution/providers.py b/lib/ansible/galaxy/dependency_resolution/providers.py
index ccb56a9d..817a1eb2 100644
--- a/lib/ansible/galaxy/dependency_resolution/providers.py
+++ b/lib/ansible/galaxy/dependency_resolution/providers.py
@@ -14,6 +14,7 @@ if t.TYPE_CHECKING:
         ConcreteArtifactsManager,
     )
     from ansible.galaxy.collection.galaxy_api_proxy import MultiGalaxyAPIProxy
+    from ansible.galaxy.api import GalaxyAPI
 
 from ansible.galaxy.collection.gpg import get_signature_from_source
 from ansible.galaxy.dependency_resolution.dataclasses import (
@@ -316,8 +317,18 @@ class CollectionDependencyProviderBase(AbstractProvider):
         # The fqcn is guaranteed to be the same
         version_req = "A SemVer-compliant version or '*' is required. See https://semver.org to learn how to compose it correctly. "
         version_req += "This is an issue with the collection."
+
+        # If we're upgrading collections, we can't calculate preinstalled_candidates until the latest matches are found.
+        # Otherwise, we can potentially avoid a Galaxy API call by doing this first.
+        preinstalled_candidates = set()
+        if not self._upgrade and first_req.type == 'galaxy':
+            preinstalled_candidates = {
+                candidate for candidate in self._preferred_candidates
+                if candidate.fqcn == fqcn and
+                all(self.is_satisfied_by(requirement, candidate) for requirement in requirements)
+            }
         try:
-            coll_versions = self._api_proxy.get_collection_versions(first_req)
+            coll_versions = [] if preinstalled_candidates else self._api_proxy.get_collection_versions(first_req)  # type: t.Iterable[t.Tuple[str, GalaxyAPI]]
         except TypeError as exc:
             if first_req.is_concrete_artifact:
                 # Non hashable versions will cause a TypeError
@@ -395,19 +406,20 @@ class CollectionDependencyProviderBase(AbstractProvider):
             reverse=True,  # prefer newer versions over older ones
         )
 
-        preinstalled_candidates = {
-            candidate for candidate in self._preferred_candidates
-            if candidate.fqcn == fqcn and
-            (
-                # check if an upgrade is necessary
-                all(self.is_satisfied_by(requirement, candidate) for requirement in requirements) and
+        if not preinstalled_candidates:
+            preinstalled_candidates = {
+                candidate for candidate in self._preferred_candidates
+                if candidate.fqcn == fqcn and
                 (
-                    not self._upgrade or
-                    # check if an upgrade is preferred
-                    all(SemanticVersion(latest.ver) <= SemanticVersion(candidate.ver) for latest in latest_matches)
+                    # check if an upgrade is necessary
+                    all(self.is_satisfied_by(requirement, candidate) for requirement in requirements) and
+                    (
+                        not self._upgrade or
+                        # check if an upgrade is preferred
+                        all(SemanticVersion(latest.ver) <= SemanticVersion(candidate.ver) for latest in latest_matches)
+                    )
                 )
-            )
-        }
+            }
 
         return list(preinstalled_candidates) + latest_matches
 
diff --git a/lib/ansible/galaxy/role.py b/lib/ansible/galaxy/role.py
index 058174c9..7a13c971 100644
--- a/lib/ansible/galaxy/role.py
+++ b/lib/ansible/galaxy/role.py
@@ -27,14 +27,16 @@ import datetime
 import os
 import tarfile
 import tempfile
-from ansible.module_utils.compat.version import LooseVersion
+
+from collections.abc import MutableSequence
 from shutil import rmtree
 
 from ansible import context
-from ansible.errors import AnsibleError
+from ansible.errors import AnsibleError, AnsibleParserError
 from ansible.galaxy.user_agent import user_agent
 from ansible.module_utils._text import to_native, to_text
 from ansible.module_utils.common.yaml import yaml_dump, yaml_load
+from ansible.module_utils.compat.version import LooseVersion
 from ansible.module_utils.urls import open_url
 from ansible.playbook.role.requirement import RoleRequirement
 from ansible.utils.display import Display
@@ -53,6 +55,7 @@ class GalaxyRole(object):
     def __init__(self, galaxy, api, name, src=None, version=None, scm=None, path=None):
 
         self._metadata = None
+        self._metadata_dependencies = None
         self._requirements = None
         self._install_info = None
         self._validate_certs = not context.CLIARGS['ignore_certs']
@@ -121,6 +124,24 @@ class GalaxyRole(object):
         return self._metadata
 
     @property
+    def metadata_dependencies(self):
+        """
+        Returns a list of dependencies from role metadata
+        """
+        if self._metadata_dependencies is None:
+            self._metadata_dependencies = []
+
+            if self.metadata is not None:
+                self._metadata_dependencies = self.metadata.get('dependencies') or []
+
+        if not isinstance(self._metadata_dependencies, MutableSequence):
+            raise AnsibleParserError(
+                f"Expected role dependencies to be a list. Role {self} has meta/main.yml with dependencies {self._metadata_dependencies}"
+            )
+
+        return self._metadata_dependencies
+
+    @property
     def install_info(self):
         """
         Returns role install info
@@ -405,4 +426,7 @@ class GalaxyRole(object):
 
                     break
 
+        if not isinstance(self._requirements, MutableSequence):
+            raise AnsibleParserError(f"Expected role dependencies to be a list. Role {self} has meta/requirements.yml {self._requirements}")
+
         return self._requirements