diff options
Diffstat (limited to 'lib/ansible/galaxy/collection/__init__.py')
| -rw-r--r-- | lib/ansible/galaxy/collection/__init__.py | 192 |
1 files changed, 124 insertions, 68 deletions
diff --git a/lib/ansible/galaxy/collection/__init__.py b/lib/ansible/galaxy/collection/__init__.py index 84444d82..60c9c94b 100644 --- a/lib/ansible/galaxy/collection/__init__.py +++ b/lib/ansible/galaxy/collection/__init__.py @@ -11,6 +11,7 @@ import fnmatch import functools import json import os +import pathlib import queue import re import shutil @@ -83,6 +84,7 @@ if t.TYPE_CHECKING: FilesManifestType = t.Dict[t.Literal['files', 'format'], t.Union[t.List[FileManifestEntryType], int]] import ansible.constants as C +from ansible.compat.importlib_resources import files from ansible.errors import AnsibleError from ansible.galaxy.api import GalaxyAPI from ansible.galaxy.collection.concrete_artifact_manager import ( @@ -122,8 +124,7 @@ from ansible.galaxy.dependency_resolution.dataclasses import ( ) from ansible.galaxy.dependency_resolution.versioning import meets_requirements from ansible.plugins.loader import get_all_plugin_loaders -from ansible.module_utils.six import raise_from -from ansible.module_utils._text import to_bytes, to_native, to_text +from ansible.module_utils.common.text.converters import to_bytes, to_native, to_text from ansible.module_utils.common.collections import is_sequence from ansible.module_utils.common.yaml import yaml_dump from ansible.utils.collection_loader import AnsibleCollectionRef @@ -282,11 +283,8 @@ def verify_local_collection(local_collection, remote_collection, artifacts_manag manifest_hash = get_hash_from_validation_source(MANIFEST_FILENAME) else: # fetch remote - b_temp_tar_path = ( # NOTE: AnsibleError is raised on URLError - artifacts_manager.get_artifact_path - if remote_collection.is_concrete_artifact - else artifacts_manager.get_galaxy_artifact_path - )(remote_collection) + # NOTE: AnsibleError is raised on URLError + b_temp_tar_path = artifacts_manager.get_artifact_path_from_unknown(remote_collection) display.vvv( u"Remote collection cached as '{path!s}'".format(path=to_text(b_temp_tar_path)) @@ -470,7 +468,7 @@ def build_collection(u_collection_path, u_output_path, force): try: collection_meta = _get_meta_from_src_dir(b_collection_path) except LookupError as lookup_err: - raise_from(AnsibleError(to_native(lookup_err)), lookup_err) + raise AnsibleError(to_native(lookup_err)) from lookup_err collection_manifest = _build_manifest(**collection_meta) file_manifest = _build_files_manifest( @@ -479,6 +477,7 @@ def build_collection(u_collection_path, u_output_path, force): collection_meta['name'], # type: ignore[arg-type] collection_meta['build_ignore'], # type: ignore[arg-type] collection_meta['manifest'], # type: ignore[arg-type] + collection_meta['license_file'], # type: ignore[arg-type] ) artifact_tarball_file_name = '{ns!s}-{name!s}-{ver!s}.tar.gz'.format( @@ -545,7 +544,7 @@ def download_collections( for fqcn, concrete_coll_pin in dep_map.copy().items(): # FIXME: move into the provider if concrete_coll_pin.is_virtual: display.display( - '{coll!s} is not downloadable'. + 'Virtual collection {coll!s} is not downloadable'. format(coll=to_text(concrete_coll_pin)), ) continue @@ -555,11 +554,7 @@ def download_collections( format(coll=to_text(concrete_coll_pin), path=to_text(b_output_path)), ) - b_src_path = ( - artifacts_manager.get_artifact_path - if concrete_coll_pin.is_concrete_artifact - else artifacts_manager.get_galaxy_artifact_path - )(concrete_coll_pin) + b_src_path = artifacts_manager.get_artifact_path_from_unknown(concrete_coll_pin) b_dest_path = os.path.join( b_output_path, @@ -659,6 +654,7 @@ def install_collections( artifacts_manager, # type: ConcreteArtifactsManager disable_gpg_verify, # type: bool offline, # type: bool + read_requirement_paths, # type: set[str] ): # type: (...) -> None """Install Ansible collections to the path specified. @@ -673,13 +669,14 @@ def install_collections( """ existing_collections = { Requirement(coll.fqcn, coll.ver, coll.src, coll.type, None) - for coll in find_existing_collections(output_path, artifacts_manager) + for path in {output_path} | read_requirement_paths + for coll in find_existing_collections(path, artifacts_manager) } unsatisfied_requirements = set( chain.from_iterable( ( - Requirement.from_dir_path(sub_coll, artifacts_manager) + Requirement.from_dir_path(to_bytes(sub_coll), artifacts_manager) for sub_coll in ( artifacts_manager. get_direct_collection_dependencies(install_req). @@ -744,7 +741,7 @@ def install_collections( for fqcn, concrete_coll_pin in dependency_map.items(): if concrete_coll_pin.is_virtual: display.vvvv( - "Encountered {coll!s}, skipping.". + "'{coll!s}' is virtual, skipping.". format(coll=to_text(concrete_coll_pin)), ) continue @@ -1065,8 +1062,9 @@ def _make_entry(name, ftype, chksum_type='sha256', chksum=None): } -def _build_files_manifest(b_collection_path, namespace, name, ignore_patterns, manifest_control): - # type: (bytes, str, str, list[str], dict[str, t.Any]) -> FilesManifestType +def _build_files_manifest(b_collection_path, namespace, name, ignore_patterns, + manifest_control, license_file): + # type: (bytes, str, str, list[str], dict[str, t.Any], t.Optional[str]) -> FilesManifestType if ignore_patterns and manifest_control is not Sentinel: raise AnsibleError('"build_ignore" and "manifest" are mutually exclusive') @@ -1076,14 +1074,15 @@ def _build_files_manifest(b_collection_path, namespace, name, ignore_patterns, m namespace, name, manifest_control, + license_file, ) return _build_files_manifest_walk(b_collection_path, namespace, name, ignore_patterns) -def _build_files_manifest_distlib(b_collection_path, namespace, name, manifest_control): - # type: (bytes, str, str, dict[str, t.Any]) -> FilesManifestType - +def _build_files_manifest_distlib(b_collection_path, namespace, name, manifest_control, + license_file): + # type: (bytes, str, str, dict[str, t.Any], t.Optional[str]) -> FilesManifestType if not HAS_DISTLIB: raise AnsibleError('Use of "manifest" requires the python "distlib" library') @@ -1116,15 +1115,20 @@ def _build_files_manifest_distlib(b_collection_path, namespace, name, manifest_c else: directives.extend([ 'include meta/*.yml', - 'include *.txt *.md *.rst COPYING LICENSE', + 'include *.txt *.md *.rst *.license COPYING LICENSE', + 'recursive-include .reuse **', + 'recursive-include LICENSES **', 'recursive-include tests **', - 'recursive-include docs **.rst **.yml **.yaml **.json **.j2 **.txt', - 'recursive-include roles **.yml **.yaml **.json **.j2', - 'recursive-include playbooks **.yml **.yaml **.json', - 'recursive-include changelogs **.yml **.yaml', - 'recursive-include plugins */**.py', + 'recursive-include docs **.rst **.yml **.yaml **.json **.j2 **.txt **.license', + 'recursive-include roles **.yml **.yaml **.json **.j2 **.license', + 'recursive-include playbooks **.yml **.yaml **.json **.license', + 'recursive-include changelogs **.yml **.yaml **.license', + 'recursive-include plugins */**.py */**.license', ]) + if license_file: + directives.append(f'include {license_file}') + plugins = set(l.package.split('.')[-1] for d, l in get_all_plugin_loaders()) for plugin in sorted(plugins): if plugin in ('modules', 'module_utils'): @@ -1135,8 +1139,8 @@ def _build_files_manifest_distlib(b_collection_path, namespace, name, manifest_c ) directives.extend([ - 'recursive-include plugins/modules **.ps1 **.yml **.yaml', - 'recursive-include plugins/module_utils **.ps1 **.psm1 **.cs', + 'recursive-include plugins/modules **.ps1 **.yml **.yaml **.license', + 'recursive-include plugins/module_utils **.ps1 **.psm1 **.cs **.license', ]) directives.extend(control.directives) @@ -1144,7 +1148,7 @@ def _build_files_manifest_distlib(b_collection_path, namespace, name, manifest_c directives.extend([ f'exclude galaxy.yml galaxy.yaml MANIFEST.json FILES.json {namespace}-{name}-*.tar.gz', 'recursive-exclude tests/output **', - 'global-exclude /.* /__pycache__', + 'global-exclude /.* /__pycache__ *.pyc *.pyo *.bak *~ *.swp', ]) display.vvv('Manifest Directives:') @@ -1321,6 +1325,8 @@ def _build_collection_tar( if os.path.islink(b_src_path): b_link_target = os.path.realpath(b_src_path) + if not os.path.exists(b_link_target): + raise AnsibleError(f"Failed to find the target path '{to_native(b_link_target)}' for the symlink '{to_native(b_src_path)}'.") if _is_child_path(b_link_target, b_collection_path): b_rel_path = os.path.relpath(b_link_target, start=os.path.dirname(b_src_path)) @@ -1375,51 +1381,101 @@ def _build_collection_dir(b_collection_path, b_collection_output, collection_man src_file = os.path.join(b_collection_path, to_bytes(file_info['name'], errors='surrogate_or_strict')) dest_file = os.path.join(b_collection_output, to_bytes(file_info['name'], errors='surrogate_or_strict')) - existing_is_exec = os.stat(src_file).st_mode & stat.S_IXUSR + existing_is_exec = os.stat(src_file, follow_symlinks=False).st_mode & stat.S_IXUSR mode = 0o0755 if existing_is_exec else 0o0644 - if os.path.isdir(src_file): + # ensure symlinks to dirs are not translated to empty dirs + if os.path.isdir(src_file) and not os.path.islink(src_file): mode = 0o0755 base_directories.append(src_file) os.mkdir(dest_file, mode) else: - shutil.copyfile(src_file, dest_file) + # do not follow symlinks to ensure the original link is used + shutil.copyfile(src_file, dest_file, follow_symlinks=False) + + # avoid setting specific permission on symlinks since it does not + # support avoid following symlinks and will thrown an exception if the + # symlink target does not exist + if not os.path.islink(dest_file): + os.chmod(dest_file, mode) - os.chmod(dest_file, mode) collection_output = to_text(b_collection_output) return collection_output -def find_existing_collections(path, artifacts_manager): +def _normalize_collection_path(path): + str_path = path.as_posix() if isinstance(path, pathlib.Path) else path + return pathlib.Path( + # This is annoying, but GalaxyCLI._resolve_path did it + os.path.expandvars(str_path) + ).expanduser().absolute() + + +def find_existing_collections(path_filter, artifacts_manager, namespace_filter=None, collection_filter=None, dedupe=True): """Locate all collections under a given path. :param path: Collection dirs layout search path. :param artifacts_manager: Artifacts manager. """ - b_path = to_bytes(path, errors='surrogate_or_strict') + if files is None: + raise AnsibleError('importlib_resources is not installed and is required') + + if path_filter and not is_sequence(path_filter): + path_filter = [path_filter] + if namespace_filter and not is_sequence(namespace_filter): + namespace_filter = [namespace_filter] + if collection_filter and not is_sequence(collection_filter): + collection_filter = [collection_filter] + + paths = set() + for path in files('ansible_collections').glob('*/*/'): + path = _normalize_collection_path(path) + if not path.is_dir(): + continue + if path_filter: + for pf in path_filter: + try: + path.relative_to(_normalize_collection_path(pf)) + except ValueError: + continue + break + else: + continue + paths.add(path) - # FIXME: consider using `glob.glob()` to simplify looping - for b_namespace in os.listdir(b_path): - b_namespace_path = os.path.join(b_path, b_namespace) - if os.path.isfile(b_namespace_path): + seen = set() + for path in paths: + namespace = path.parent.name + name = path.name + if namespace_filter and namespace not in namespace_filter: + continue + if collection_filter and name not in collection_filter: continue - # FIXME: consider feeding b_namespace_path to Candidate.from_dir_path to get subdirs automatically - for b_collection in os.listdir(b_namespace_path): - b_collection_path = os.path.join(b_namespace_path, b_collection) - if not os.path.isdir(b_collection_path): + if dedupe: + try: + collection_path = files(f'ansible_collections.{namespace}.{name}') + except ImportError: continue + if collection_path in seen: + continue + seen.add(collection_path) + else: + collection_path = path - try: - req = Candidate.from_dir_path_as_unknown(b_collection_path, artifacts_manager) - except ValueError as val_err: - raise_from(AnsibleError(val_err), val_err) + b_collection_path = to_bytes(collection_path.as_posix()) - display.vvv( - u"Found installed collection {coll!s} at '{path!s}'". - format(coll=to_text(req), path=to_text(req.src)) - ) - yield req + try: + req = Candidate.from_dir_path_as_unknown(b_collection_path, artifacts_manager) + except ValueError as val_err: + display.warning(f'{val_err}') + continue + + display.vvv( + u"Found installed collection {coll!s} at '{path!s}'". + format(coll=to_text(req), path=to_text(req.src)) + ) + yield req def install(collection, path, artifacts_manager): # FIXME: mv to dataclasses? @@ -1430,10 +1486,7 @@ def install(collection, path, artifacts_manager): # FIXME: mv to dataclasses? :param path: Collection dirs layout path. :param artifacts_manager: Artifacts manager. """ - b_artifact_path = ( - artifacts_manager.get_artifact_path if collection.is_concrete_artifact - else artifacts_manager.get_galaxy_artifact_path - )(collection) + b_artifact_path = artifacts_manager.get_artifact_path_from_unknown(collection) collection_path = os.path.join(path, collection.namespace, collection.name) b_collection_path = to_bytes(collection_path, errors='surrogate_or_strict') @@ -1587,6 +1640,7 @@ def install_src(collection, b_collection_path, b_collection_output_path, artifac collection_meta['namespace'], collection_meta['name'], collection_meta['build_ignore'], collection_meta['manifest'], + collection_meta['license_file'], ) collection_output_path = _build_collection_dir( @@ -1763,10 +1817,15 @@ def _resolve_depenency_map( elif not req.specifier.contains(RESOLVELIB_VERSION.vstring): raise AnsibleError(f"ansible-galaxy requires {req.name}{req.specifier}") + pre_release_hint = '' if allow_pre_release else ( + 'Hint: Pre-releases hosted on Galaxy or Automation Hub are not ' + 'installed by default unless a specific version is requested. ' + 'To enable pre-releases globally, use --pre.' + ) + collection_dep_resolver = build_collection_dependency_resolver( galaxy_apis=galaxy_apis, concrete_artifacts_manager=concrete_artifacts_manager, - user_requirements=requested_requirements, preferred_candidates=preferred_candidates, with_deps=not no_deps, with_pre_releases=allow_pre_release, @@ -1798,13 +1857,12 @@ def _resolve_depenency_map( ), conflict_causes, )) - raise raise_from( # NOTE: Leading "raise" is a hack for mypy bug #9717 - AnsibleError('\n'.join(error_msg_lines)), - dep_exc, - ) + error_msg_lines.append(pre_release_hint) + raise AnsibleError('\n'.join(error_msg_lines)) from dep_exc except CollectionDependencyInconsistentCandidate as dep_exc: parents = [ - str(p) for p in dep_exc.criterion.iter_parent() + "%s.%s:%s" % (p.namespace, p.name, p.ver) + for p in dep_exc.criterion.iter_parent() if p is not None ] @@ -1826,10 +1884,8 @@ def _resolve_depenency_map( error_msg_lines.append( '* {req.fqcn!s}:{req.ver!s}'.format(req=req) ) + error_msg_lines.append(pre_release_hint) - raise raise_from( # NOTE: Leading "raise" is a hack for mypy bug #9717 - AnsibleError('\n'.join(error_msg_lines)), - dep_exc, - ) + raise AnsibleError('\n'.join(error_msg_lines)) from dep_exc except ValueError as exc: raise AnsibleError(to_native(exc)) from exc |