diff options
Diffstat (limited to 'lib/ansible/galaxy/dependency_resolution')
4 files changed, 120 insertions, 89 deletions
diff --git a/lib/ansible/galaxy/dependency_resolution/__init__.py b/lib/ansible/galaxy/dependency_resolution/__init__.py index cfde7df0..eeffd299 100644 --- a/lib/ansible/galaxy/dependency_resolution/__init__.py +++ b/lib/ansible/galaxy/dependency_resolution/__init__.py @@ -13,10 +13,7 @@ if t.TYPE_CHECKING: from ansible.galaxy.collection.concrete_artifact_manager import ( ConcreteArtifactsManager, ) - from ansible.galaxy.dependency_resolution.dataclasses import ( - Candidate, - Requirement, - ) + from ansible.galaxy.dependency_resolution.dataclasses import Candidate from ansible.galaxy.collection.galaxy_api_proxy import MultiGalaxyAPIProxy from ansible.galaxy.dependency_resolution.providers import CollectionDependencyProvider @@ -27,7 +24,6 @@ from ansible.galaxy.dependency_resolution.resolvers import CollectionDependencyR def build_collection_dependency_resolver( galaxy_apis, # type: t.Iterable[GalaxyAPI] concrete_artifacts_manager, # type: ConcreteArtifactsManager - user_requirements, # type: t.Iterable[Requirement] preferred_candidates=None, # type: t.Iterable[Candidate] with_deps=True, # type: bool with_pre_releases=False, # type: bool @@ -44,7 +40,6 @@ def build_collection_dependency_resolver( CollectionDependencyProvider( apis=MultiGalaxyAPIProxy(galaxy_apis, concrete_artifacts_manager, offline=offline), concrete_artifacts_manager=concrete_artifacts_manager, - user_requirements=user_requirements, preferred_candidates=preferred_candidates, with_deps=with_deps, with_pre_releases=with_pre_releases, diff --git a/lib/ansible/galaxy/dependency_resolution/dataclasses.py b/lib/ansible/galaxy/dependency_resolution/dataclasses.py index 35b65054..7e8fb57a 100644 --- a/lib/ansible/galaxy/dependency_resolution/dataclasses.py +++ b/lib/ansible/galaxy/dependency_resolution/dataclasses.py @@ -29,7 +29,8 @@ if t.TYPE_CHECKING: from ansible.errors import AnsibleError, AnsibleAssertionError from ansible.galaxy.api import GalaxyAPI -from ansible.module_utils._text import to_bytes, to_native, to_text +from ansible.galaxy.collection import HAS_PACKAGING, PkgReq +from ansible.module_utils.common.text.converters import to_bytes, to_native, to_text from ansible.module_utils.common.arg_spec import ArgumentSpecValidator from ansible.utils.collection_loader import AnsibleCollectionRef from ansible.utils.display import Display @@ -215,10 +216,15 @@ class _ComputedReqKindsMixin: return cls.from_dir_path_implicit(dir_path) @classmethod - def from_dir_path(cls, dir_path, art_mgr): + def from_dir_path( # type: ignore[misc] + cls, # type: t.Type[Collection] + dir_path, # type: bytes + art_mgr, # type: ConcreteArtifactsManager + ): # type: (...) -> Collection """Make collection from an directory with metadata.""" - b_dir_path = to_bytes(dir_path, errors='surrogate_or_strict') - if not _is_collection_dir(b_dir_path): + if dir_path.endswith(to_bytes(os.path.sep)): + dir_path = dir_path.rstrip(to_bytes(os.path.sep)) + if not _is_collection_dir(dir_path): display.warning( u"Collection at '{path!s}' does not have a {manifest_json!s} " u'file, nor has it {galaxy_yml!s}: cannot detect version.'. @@ -267,6 +273,8 @@ class _ComputedReqKindsMixin: regardless of whether any of known metadata files are present. """ # There is no metadata, but it isn't required for a functional collection. Determine the namespace.name from the path. + if dir_path.endswith(to_bytes(os.path.sep)): + dir_path = dir_path.rstrip(to_bytes(os.path.sep)) u_dir_path = to_text(dir_path, errors='surrogate_or_strict') path_list = u_dir_path.split(os.path.sep) req_name = '.'.join(path_list[-2:]) @@ -275,13 +283,25 @@ class _ComputedReqKindsMixin: @classmethod def from_string(cls, collection_input, artifacts_manager, supplemental_signatures): req = {} - if _is_concrete_artifact_pointer(collection_input): - # Arg is a file path or URL to a collection + if _is_concrete_artifact_pointer(collection_input) or AnsibleCollectionRef.is_valid_collection_name(collection_input): + # Arg is a file path or URL to a collection, or just a collection req['name'] = collection_input - else: + elif ':' in collection_input: req['name'], _sep, req['version'] = collection_input.partition(':') if not req['version']: del req['version'] + else: + if not HAS_PACKAGING: + raise AnsibleError("Failed to import packaging, check that a supported version is installed") + try: + pkg_req = PkgReq(collection_input) + except Exception as e: + # packaging doesn't know what this is, let it fly, better errors happen in from_requirement_dict + req['name'] = collection_input + else: + req['name'] = pkg_req.name + if pkg_req.specifier: + req['version'] = to_text(pkg_req.specifier) req['signatures'] = supplemental_signatures return cls.from_requirement_dict(req, artifacts_manager) @@ -414,6 +434,9 @@ class _ComputedReqKindsMixin: format(not_url=req_source.api_server), ) + if req_type == 'dir' and req_source.endswith(os.path.sep): + req_source = req_source.rstrip(os.path.sep) + tmp_inst_req = cls(req_name, req_version, req_source, req_type, req_signature_sources) if req_type not in {'galaxy', 'subdirs'} and req_name is None: @@ -440,8 +463,8 @@ class _ComputedReqKindsMixin: def __unicode__(self): if self.fqcn is None: return ( - f'{self.type} collection from a Git repo' if self.is_scm - else f'{self.type} collection from a namespace' + u'"virtual collection Git repo"' if self.is_scm + else u'"virtual collection namespace"' ) return ( @@ -481,14 +504,14 @@ class _ComputedReqKindsMixin: @property def namespace(self): if self.is_virtual: - raise TypeError(f'{self.type} collections do not have a namespace') + raise TypeError('Virtual collections do not have a namespace') return self._get_separate_ns_n_name()[0] @property def name(self): if self.is_virtual: - raise TypeError(f'{self.type} collections do not have a name') + raise TypeError('Virtual collections do not have a name') return self._get_separate_ns_n_name()[-1] @@ -542,6 +565,27 @@ class _ComputedReqKindsMixin: return not self.is_concrete_artifact @property + def is_pinned(self): + """Indicate if the version set is considered pinned. + + This essentially computes whether the version field of the current + requirement explicitly requests a specific version and not an allowed + version range. + + It is then used to help the resolvelib-based dependency resolver judge + whether it's acceptable to consider a pre-release candidate version + despite pre-release installs not being requested by the end-user + explicitly. + + See https://github.com/ansible/ansible/pull/81606 for extra context. + """ + version_string = self.ver[0] + return version_string.isdigit() or not ( + version_string == '*' or + version_string.startswith(('<', '>', '!=')) + ) + + @property def source_info(self): return self._source_info diff --git a/lib/ansible/galaxy/dependency_resolution/errors.py b/lib/ansible/galaxy/dependency_resolution/errors.py index ae3b4396..acd88575 100644 --- a/lib/ansible/galaxy/dependency_resolution/errors.py +++ b/lib/ansible/galaxy/dependency_resolution/errors.py @@ -7,7 +7,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type try: - from resolvelib.resolvers import ( + from resolvelib.resolvers import ( # pylint: disable=unused-import ResolutionImpossible as CollectionDependencyResolutionImpossible, InconsistentCandidate as CollectionDependencyInconsistentCandidate, ) diff --git a/lib/ansible/galaxy/dependency_resolution/providers.py b/lib/ansible/galaxy/dependency_resolution/providers.py index 6ad1de84..f13d3ecf 100644 --- a/lib/ansible/galaxy/dependency_resolution/providers.py +++ b/lib/ansible/galaxy/dependency_resolution/providers.py @@ -40,7 +40,7 @@ except ImportError: # TODO: add python requirements to ansible-test's ansible-core distribution info and remove the hardcoded lowerbound/upperbound fallback RESOLVELIB_LOWERBOUND = SemanticVersion("0.5.3") -RESOLVELIB_UPPERBOUND = SemanticVersion("0.9.0") +RESOLVELIB_UPPERBOUND = SemanticVersion("1.1.0") RESOLVELIB_VERSION = SemanticVersion.from_loose_version(LooseVersion(resolvelib_version)) @@ -51,7 +51,6 @@ class CollectionDependencyProviderBase(AbstractProvider): self, # type: CollectionDependencyProviderBase apis, # type: MultiGalaxyAPIProxy concrete_artifacts_manager=None, # type: ConcreteArtifactsManager - user_requirements=None, # type: t.Iterable[Requirement] preferred_candidates=None, # type: t.Iterable[Candidate] with_deps=True, # type: bool with_pre_releases=False, # type: bool @@ -87,58 +86,12 @@ class CollectionDependencyProviderBase(AbstractProvider): Requirement.from_requirement_dict, art_mgr=concrete_artifacts_manager, ) - self._pinned_candidate_requests = set( - # NOTE: User-provided signatures are supplemental, so signatures - # NOTE: are not used to determine if a candidate is user-requested - Candidate(req.fqcn, req.ver, req.src, req.type, None) - for req in (user_requirements or ()) - if req.is_concrete_artifact or ( - req.ver != '*' and - not req.ver.startswith(('<', '>', '!=')) - ) - ) self._preferred_candidates = set(preferred_candidates or ()) self._with_deps = with_deps self._with_pre_releases = with_pre_releases self._upgrade = upgrade self._include_signatures = include_signatures - def _is_user_requested(self, candidate): # type: (Candidate) -> bool - """Check if the candidate is requested by the user.""" - if candidate in self._pinned_candidate_requests: - return True - - if candidate.is_online_index_pointer and candidate.src is not None: - # NOTE: Candidate is a namedtuple, it has a source server set - # NOTE: to a specific GalaxyAPI instance or `None`. When the - # NOTE: user runs - # NOTE: - # NOTE: $ ansible-galaxy collection install ns.coll - # NOTE: - # NOTE: then it's saved in `self._pinned_candidate_requests` - # NOTE: as `('ns.coll', '*', None, 'galaxy')` but then - # NOTE: `self.find_matches()` calls `self.is_satisfied_by()` - # NOTE: with Candidate instances bound to each specific - # NOTE: server available, those look like - # NOTE: `('ns.coll', '*', GalaxyAPI(...), 'galaxy')` and - # NOTE: wouldn't match the user requests saved in - # NOTE: `self._pinned_candidate_requests`. This is why we - # NOTE: normalize the collection to have `src=None` and try - # NOTE: again. - # NOTE: - # NOTE: When the user request comes from `requirements.yml` - # NOTE: with the `source:` set, it'll match the first check - # NOTE: but it still can have entries with `src=None` so this - # NOTE: normalized check is still necessary. - # NOTE: - # NOTE: User-provided signatures are supplemental, so signatures - # NOTE: are not used to determine if a candidate is user-requested - return Candidate( - candidate.fqcn, candidate.ver, None, candidate.type, None - ) in self._pinned_candidate_requests - - return False - def identify(self, requirement_or_candidate): # type: (t.Union[Candidate, Requirement]) -> str """Given requirement or candidate, return an identifier for it. @@ -190,7 +143,7 @@ class CollectionDependencyProviderBase(AbstractProvider): Mapping of identifier, list of named tuple pairs. The named tuples have the entries ``requirement`` and ``parent``. - resolvelib >=0.8.0, <= 0.8.1 + resolvelib >=0.8.0, <= 1.0.1 :param identifier: The value returned by ``identify()``. @@ -342,25 +295,79 @@ class CollectionDependencyProviderBase(AbstractProvider): latest_matches = [] signatures = [] extra_signature_sources = [] # type: list[str] + + discarding_pre_releases_acceptable = any( + not is_pre_release(candidate_version) + for candidate_version, _src_server in coll_versions + ) + + # NOTE: The optimization of conditionally looping over the requirements + # NOTE: is used to skip having to compute the pinned status of all + # NOTE: requirements and apply version normalization to the found ones. + all_pinned_requirement_version_numbers = { + # NOTE: Pinned versions can start with a number, but also with an + # NOTE: equals sign. Stripping it at the beginning should be + # NOTE: enough. If there's a space after equals, the second strip + # NOTE: will take care of it. + # NOTE: Without this conversion, requirements versions like + # NOTE: '1.2.3-alpha.4' work, but '=1.2.3-alpha.4' don't. + requirement.ver.lstrip('=').strip() + for requirement in requirements + if requirement.is_pinned + } if discarding_pre_releases_acceptable else set() + for version, src_server in coll_versions: tmp_candidate = Candidate(fqcn, version, src_server, 'galaxy', None) - unsatisfied = False for requirement in requirements: - unsatisfied |= not self.is_satisfied_by(requirement, tmp_candidate) + candidate_satisfies_requirement = self.is_satisfied_by( + requirement, tmp_candidate, + ) + if not candidate_satisfies_requirement: + break + + should_disregard_pre_release_candidate = ( + # NOTE: Do not discard pre-release candidates in the + # NOTE: following cases: + # NOTE: * the end-user requested pre-releases explicitly; + # NOTE: * the candidate is a concrete artifact (e.g. a + # NOTE: Git repository, subdirs, a tarball URL, or a + # NOTE: local dir or file etc.); + # NOTE: * the candidate's pre-release version exactly + # NOTE: matches a version specifically requested by one + # NOTE: of the requirements in the current match + # NOTE: discovery round (i.e. matching a requirement + # NOTE: that is not a range but an explicit specific + # NOTE: version pin). This works when some requirements + # NOTE: request version ranges but others (possibly on + # NOTE: different dependency tree level depths) demand + # NOTE: pre-release dependency versions, even if those + # NOTE: dependencies are transitive. + is_pre_release(tmp_candidate.ver) + and discarding_pre_releases_acceptable + and not ( + self._with_pre_releases + or tmp_candidate.is_concrete_artifact + or version in all_pinned_requirement_version_numbers + ) + ) + if should_disregard_pre_release_candidate: + break + # FIXME - # unsatisfied |= not self.is_satisfied_by(requirement, tmp_candidate) or not ( - # requirement.src is None or # if this is true for some candidates but not all it will break key param - Nonetype can't be compared to str + # candidate_is_from_requested_source = ( + # requirement.src is None # if this is true for some candidates but not all it will break key param - Nonetype can't be compared to str # or requirement.src == candidate.src # ) - if unsatisfied: - break + # if not candidate_is_from_requested_source: + # break + if not self._include_signatures: continue extra_signature_sources.extend(requirement.signature_sources or []) - if not unsatisfied: + else: # candidate satisfies requirements, `break` never happened if self._include_signatures: for extra_source in extra_signature_sources: signatures.append(get_signature_from_source(extra_source)) @@ -405,21 +412,6 @@ class CollectionDependencyProviderBase(AbstractProvider): :returns: Indication whether the `candidate` is a viable \ solution to the `requirement`. """ - # NOTE: Only allow pre-release candidates if we want pre-releases - # NOTE: or the req ver was an exact match with the pre-release - # NOTE: version. Another case where we'd want to allow - # NOTE: pre-releases is when there are several user requirements - # NOTE: and one of them is a pre-release that also matches a - # NOTE: transitive dependency of another requirement. - allow_pre_release = self._with_pre_releases or not ( - requirement.ver == '*' or - requirement.ver.startswith('<') or - requirement.ver.startswith('>') or - requirement.ver.startswith('!=') - ) or self._is_user_requested(candidate) - if is_pre_release(candidate.ver) and not allow_pre_release: - return False - # NOTE: This is a set of Pipenv-inspired optimizations. Ref: # https://github.com/sarugaku/passa/blob/2ac00f1/src/passa/models/providers.py#L58-L74 if ( |