diff options
Diffstat (limited to 'bin/ansible-galaxy')
| -rwxr-xr-x | bin/ansible-galaxy | 273 |
1 files changed, 125 insertions, 148 deletions
diff --git a/bin/ansible-galaxy b/bin/ansible-galaxy index 334e4bf4..536964e2 100755 --- a/bin/ansible-galaxy +++ b/bin/ansible-galaxy @@ -10,11 +10,9 @@ __metaclass__ = type # ansible.cli needs to be imported first, to ensure the source bin/* scripts run that code first from ansible.cli import CLI -import argparse import functools import json import os.path -import pathlib import re import shutil import sys @@ -53,7 +51,7 @@ from ansible.galaxy.token import BasicAuthToken, GalaxyToken, KeycloakToken, NoT from ansible.module_utils.ansible_release import __version__ as ansible_version from ansible.module_utils.common.collections import is_iterable from ansible.module_utils.common.yaml import yaml_dump, yaml_load -from ansible.module_utils.common.text.converters import to_bytes, to_native, to_text +from ansible.module_utils._text import to_bytes, to_native, to_text from ansible.module_utils import six from ansible.parsing.dataloader import DataLoader from ansible.parsing.yaml.loader import AnsibleLoader @@ -73,7 +71,7 @@ SERVER_DEF = [ ('password', False, 'str'), ('token', False, 'str'), ('auth_url', False, 'str'), - ('api_version', False, 'int'), + ('v3', False, 'bool'), ('validate_certs', False, 'bool'), ('client_id', False, 'str'), ('timeout', False, 'int'), @@ -81,9 +79,9 @@ SERVER_DEF = [ # config definition fields SERVER_ADDITIONAL = { - 'api_version': {'default': None, 'choices': [2, 3]}, + 'v3': {'default': 'False'}, 'validate_certs': {'cli': [{'name': 'validate_certs'}]}, - 'timeout': {'default': C.GALAXY_SERVER_TIMEOUT, 'cli': [{'name': 'timeout'}]}, + 'timeout': {'default': '60', 'cli': [{'name': 'timeout'}]}, 'token': {'default': None}, } @@ -101,8 +99,7 @@ def with_collection_artifacts_manager(wrapped_method): return wrapped_method(*args, **kwargs) # FIXME: use validate_certs context from Galaxy servers when downloading collections - # .get used here for when this is used in a non-CLI context - artifacts_manager_kwargs = {'validate_certs': context.CLIARGS.get('resolved_validate_certs', True)} + artifacts_manager_kwargs = {'validate_certs': context.CLIARGS['resolved_validate_certs']} keyring = context.CLIARGS.get('keyring', None) if keyring is not None: @@ -159,8 +156,8 @@ def _get_collection_widths(collections): fqcn_set = {to_text(c.fqcn) for c in collections} version_set = {to_text(c.ver) for c in collections} - fqcn_length = len(max(fqcn_set or [''], key=len)) - version_length = len(max(version_set or [''], key=len)) + fqcn_length = len(max(fqcn_set, key=len)) + version_length = len(max(version_set, key=len)) return fqcn_length, version_length @@ -241,49 +238,45 @@ class GalaxyCLI(CLI): ) # Common arguments that apply to more than 1 action - common = opt_help.ArgumentParser(add_help=False) + common = opt_help.argparse.ArgumentParser(add_help=False) common.add_argument('-s', '--server', dest='api_server', help='The Galaxy API server URL') - common.add_argument('--api-version', type=int, choices=[2, 3], help=argparse.SUPPRESS) # Hidden argument that should only be used in our tests common.add_argument('--token', '--api-key', dest='api_key', help='The Ansible Galaxy API key which can be found at ' 'https://galaxy.ansible.com/me/preferences.') common.add_argument('-c', '--ignore-certs', action='store_true', dest='ignore_certs', help='Ignore SSL certificate validation errors.', default=None) - - # --timeout uses the default None to handle two different scenarios. - # * --timeout > C.GALAXY_SERVER_TIMEOUT for non-configured servers - # * --timeout > server-specific timeout > C.GALAXY_SERVER_TIMEOUT for configured servers. - common.add_argument('--timeout', dest='timeout', type=int, + common.add_argument('--timeout', dest='timeout', type=int, default=60, help="The time to wait for operations against the galaxy server, defaults to 60s.") opt_help.add_verbosity_options(common) - force = opt_help.ArgumentParser(add_help=False) + force = opt_help.argparse.ArgumentParser(add_help=False) force.add_argument('-f', '--force', dest='force', action='store_true', default=False, help='Force overwriting an existing role or collection') - github = opt_help.ArgumentParser(add_help=False) + github = opt_help.argparse.ArgumentParser(add_help=False) github.add_argument('github_user', help='GitHub username') github.add_argument('github_repo', help='GitHub repository') - offline = opt_help.ArgumentParser(add_help=False) + offline = opt_help.argparse.ArgumentParser(add_help=False) offline.add_argument('--offline', dest='offline', default=False, action='store_true', help="Don't query the galaxy API when creating roles") default_roles_path = C.config.get_configuration_definition('DEFAULT_ROLES_PATH').get('default', '') - roles_path = opt_help.ArgumentParser(add_help=False) + roles_path = opt_help.argparse.ArgumentParser(add_help=False) roles_path.add_argument('-p', '--roles-path', dest='roles_path', type=opt_help.unfrack_path(pathsep=True), default=C.DEFAULT_ROLES_PATH, action=opt_help.PrependListAction, help='The path to the directory containing your roles. The default is the first ' 'writable one configured via DEFAULT_ROLES_PATH: %s ' % default_roles_path) - collections_path = opt_help.ArgumentParser(add_help=False) + collections_path = opt_help.argparse.ArgumentParser(add_help=False) collections_path.add_argument('-p', '--collections-path', dest='collections_path', type=opt_help.unfrack_path(pathsep=True), + default=AnsibleCollectionConfig.collection_paths, action=opt_help.PrependListAction, help="One or more directories to search for collections in addition " "to the default COLLECTIONS_PATHS. Separate multiple paths " "with '{0}'.".format(os.path.pathsep)) - cache_options = opt_help.ArgumentParser(add_help=False) + cache_options = opt_help.argparse.ArgumentParser(add_help=False) cache_options.add_argument('--clear-response-cache', dest='clear_response_cache', action='store_true', default=False, help='Clear the existing server response cache.') cache_options.add_argument('--no-cache', dest='no_cache', action='store_true', default=False, @@ -467,15 +460,12 @@ class GalaxyCLI(CLI): valid_signature_count_help = 'The number of signatures that must successfully verify the collection. This should be a positive integer ' \ 'or all to signify that all signatures must be used to verify the collection. ' \ 'Prepend the value with + to fail if no valid signatures are found for the collection (e.g. +all).' - ignore_gpg_status_help = 'A space separated list of status codes to ignore during signature verification (for example, NO_PUBKEY FAILURE). ' \ - 'Descriptions for the choices can be seen at L(https://github.com/gpg/gnupg/blob/master/doc/DETAILS#general-status-codes).' \ - 'Note: specify these after positional arguments or use -- to separate them.' + ignore_gpg_status_help = 'A status code to ignore during signature verification (for example, NO_PUBKEY). ' \ + 'Provide this option multiple times to ignore a list of status codes. ' \ + 'Descriptions for the choices can be seen at L(https://github.com/gpg/gnupg/blob/master/doc/DETAILS#general-status-codes).' verify_parser.add_argument('--required-valid-signature-count', dest='required_valid_signature_count', type=validate_signature_count, help=valid_signature_count_help, default=C.GALAXY_REQUIRED_VALID_SIGNATURE_COUNT) verify_parser.add_argument('--ignore-signature-status-code', dest='ignore_gpg_errors', type=str, action='append', - help=opt_help.argparse.SUPPRESS, default=C.GALAXY_IGNORE_INVALID_SIGNATURE_STATUS_CODES, - choices=list(GPG_ERROR_MAP.keys())) - verify_parser.add_argument('--ignore-signature-status-codes', dest='ignore_gpg_errors', type=str, action='extend', nargs='+', help=ignore_gpg_status_help, default=C.GALAXY_IGNORE_INVALID_SIGNATURE_STATUS_CODES, choices=list(GPG_ERROR_MAP.keys())) @@ -511,9 +501,9 @@ class GalaxyCLI(CLI): valid_signature_count_help = 'The number of signatures that must successfully verify the collection. This should be a positive integer ' \ 'or -1 to signify that all signatures must be used to verify the collection. ' \ 'Prepend the value with + to fail if no valid signatures are found for the collection (e.g. +all).' - ignore_gpg_status_help = 'A space separated list of status codes to ignore during signature verification (for example, NO_PUBKEY FAILURE). ' \ - 'Descriptions for the choices can be seen at L(https://github.com/gpg/gnupg/blob/master/doc/DETAILS#general-status-codes).' \ - 'Note: specify these after positional arguments or use -- to separate them.' + ignore_gpg_status_help = 'A status code to ignore during signature verification (for example, NO_PUBKEY). ' \ + 'Provide this option multiple times to ignore a list of status codes. ' \ + 'Descriptions for the choices can be seen at L(https://github.com/gpg/gnupg/blob/master/doc/DETAILS#general-status-codes).' if galaxy_type == 'collection': install_parser.add_argument('-p', '--collections-path', dest='collections_path', @@ -537,9 +527,6 @@ class GalaxyCLI(CLI): install_parser.add_argument('--required-valid-signature-count', dest='required_valid_signature_count', type=validate_signature_count, help=valid_signature_count_help, default=C.GALAXY_REQUIRED_VALID_SIGNATURE_COUNT) install_parser.add_argument('--ignore-signature-status-code', dest='ignore_gpg_errors', type=str, action='append', - help=opt_help.argparse.SUPPRESS, default=C.GALAXY_IGNORE_INVALID_SIGNATURE_STATUS_CODES, - choices=list(GPG_ERROR_MAP.keys())) - install_parser.add_argument('--ignore-signature-status-codes', dest='ignore_gpg_errors', type=str, action='extend', nargs='+', help=ignore_gpg_status_help, default=C.GALAXY_IGNORE_INVALID_SIGNATURE_STATUS_CODES, choices=list(GPG_ERROR_MAP.keys())) install_parser.add_argument('--offline', dest='offline', action='store_true', default=False, @@ -564,9 +551,6 @@ class GalaxyCLI(CLI): install_parser.add_argument('--required-valid-signature-count', dest='required_valid_signature_count', type=validate_signature_count, help=valid_signature_count_help, default=C.GALAXY_REQUIRED_VALID_SIGNATURE_COUNT) install_parser.add_argument('--ignore-signature-status-code', dest='ignore_gpg_errors', type=str, action='append', - help=opt_help.argparse.SUPPRESS, default=C.GALAXY_IGNORE_INVALID_SIGNATURE_STATUS_CODES, - choices=list(GPG_ERROR_MAP.keys())) - install_parser.add_argument('--ignore-signature-status-codes', dest='ignore_gpg_errors', type=str, action='extend', nargs='+', help=ignore_gpg_status_help, default=C.GALAXY_IGNORE_INVALID_SIGNATURE_STATUS_CODES, choices=list(GPG_ERROR_MAP.keys())) @@ -638,7 +622,7 @@ class GalaxyCLI(CLI): return config_def galaxy_options = {} - for optional_key in ['clear_response_cache', 'no_cache']: + for optional_key in ['clear_response_cache', 'no_cache', 'timeout']: if optional_key in context.CLIARGS: galaxy_options[optional_key] = context.CLIARGS[optional_key] @@ -663,22 +647,17 @@ class GalaxyCLI(CLI): client_id = server_options.pop('client_id') token_val = server_options['token'] or NoTokenSentinel username = server_options['username'] - api_version = server_options.pop('api_version') + v3 = server_options.pop('v3') if server_options['validate_certs'] is None: server_options['validate_certs'] = context.CLIARGS['resolved_validate_certs'] validate_certs = server_options['validate_certs'] - # This allows a user to explicitly force use of an API version when - # multiple versions are supported. This was added for testing - # against pulp_ansible and I'm not sure it has a practical purpose - # outside of this use case. As such, this option is not documented - # as of now - if api_version: - display.warning( - f'The specified "api_version" configuration for the galaxy server "{server_key}" is ' - 'not a public configuration, and may be removed at any time without warning.' - ) - server_options['available_api_versions'] = {'v%s' % api_version: '/v%s' % api_version} + if v3: + # This allows a user to explicitly indicate the server uses the /v3 API + # This was added for testing against pulp_ansible and I'm not sure it has + # a practical purpose outside of this use case. As such, this option is not + # documented as of now + server_options['available_api_versions'] = {'v3': '/v3'} # default case if no auth info is provided. server_options['token'] = None @@ -704,17 +683,9 @@ class GalaxyCLI(CLI): )) cmd_server = context.CLIARGS['api_server'] - if context.CLIARGS['api_version']: - api_version = context.CLIARGS['api_version'] - display.warning( - 'The --api-version is not a public argument, and may be removed at any time without warning.' - ) - galaxy_options['available_api_versions'] = {'v%s' % api_version: '/v%s' % api_version} - cmd_token = GalaxyToken(token=context.CLIARGS['api_key']) validate_certs = context.CLIARGS['resolved_validate_certs'] - default_server_timeout = context.CLIARGS['timeout'] if context.CLIARGS['timeout'] is not None else C.GALAXY_SERVER_TIMEOUT if cmd_server: # Cmd args take precedence over the config entry but fist check if the arg was a name and use that config # entry, otherwise create a new API entry for the server specified. @@ -726,7 +697,6 @@ class GalaxyCLI(CLI): self.galaxy, 'cmd_arg', cmd_server, token=cmd_token, priority=len(config_servers) + 1, validate_certs=validate_certs, - timeout=default_server_timeout, **galaxy_options )) else: @@ -738,7 +708,6 @@ class GalaxyCLI(CLI): self.galaxy, 'default', C.GALAXY_SERVER, token=cmd_token, priority=0, validate_certs=validate_certs, - timeout=default_server_timeout, **galaxy_options )) @@ -835,7 +804,7 @@ class GalaxyCLI(CLI): for role_req in file_requirements: requirements['roles'] += parse_role_req(role_req) - elif isinstance(file_requirements, dict): + else: # Newer format with a collections and/or roles key extra_keys = set(file_requirements.keys()).difference(set(['roles', 'collections'])) if extra_keys: @@ -854,9 +823,6 @@ class GalaxyCLI(CLI): for collection_req in file_requirements.get('collections') or [] ] - else: - raise AnsibleError(f"Expecting requirements yaml to be a list or dictionary but got {type(file_requirements).__name__}") - return requirements def _init_coll_req_dict(self, coll_req): @@ -1220,16 +1186,11 @@ class GalaxyCLI(CLI): df.write(b_rendered) else: f_rel_path = os.path.relpath(os.path.join(root, f), obj_skeleton) - shutil.copyfile(os.path.join(root, f), os.path.join(obj_path, f_rel_path), follow_symlinks=False) + shutil.copyfile(os.path.join(root, f), os.path.join(obj_path, f_rel_path)) for d in dirs: b_dir_path = to_bytes(os.path.join(obj_path, rel_root, d), errors='surrogate_or_strict') - if os.path.exists(b_dir_path): - continue - b_src_dir = to_bytes(os.path.join(root, d), errors='surrogate_or_strict') - if os.path.islink(b_src_dir): - shutil.copyfile(b_src_dir, b_dir_path, follow_symlinks=False) - else: + if not os.path.exists(b_dir_path): os.makedirs(b_dir_path) display.display("- %s %s was created successfully" % (galaxy_type.title(), obj_name)) @@ -1293,7 +1254,7 @@ class GalaxyCLI(CLI): """Compare checksums with the collection(s) found on the server and the installed copy. This does not verify dependencies.""" collections = context.CLIARGS['args'] - search_paths = AnsibleCollectionConfig.collection_paths + search_paths = context.CLIARGS['collections_path'] ignore_errors = context.CLIARGS['ignore_errors'] local_verify_only = context.CLIARGS['offline'] requirements_file = context.CLIARGS['requirements'] @@ -1433,19 +1394,7 @@ class GalaxyCLI(CLI): upgrade = context.CLIARGS.get('upgrade', False) collections_path = C.COLLECTIONS_PATHS - - managed_paths = set(validate_collection_path(p) for p in C.COLLECTIONS_PATHS) - read_req_paths = set(validate_collection_path(p) for p in AnsibleCollectionConfig.collection_paths) - - unexpected_path = C.GALAXY_COLLECTIONS_PATH_WARNING and not any(p.startswith(path) for p in managed_paths) - if unexpected_path and any(p.startswith(path) for p in read_req_paths): - display.warning( - f"The specified collections path '{path}' appears to be part of the pip Ansible package. " - "Managing these directly with ansible-galaxy could break the Ansible package. " - "Install collections to a configured collections path, which will take precedence over " - "collections found in the PYTHONPATH." - ) - elif unexpected_path: + if len([p for p in collections_path if p.startswith(path)]) == 0: display.warning("The specified collections path '%s' is not part of the configured Ansible " "collections paths '%s'. The installed collection will not be picked up in an Ansible " "run, unless within a playbook-adjacent collections directory." % (to_text(path), to_text(":".join(collections_path)))) @@ -1462,7 +1411,6 @@ class GalaxyCLI(CLI): artifacts_manager=artifacts_manager, disable_gpg_verify=disable_gpg_verify, offline=context.CLIARGS.get('offline', False), - read_requirement_paths=read_req_paths, ) return 0 @@ -1631,9 +1579,7 @@ class GalaxyCLI(CLI): display.warning(w) if not path_found: - raise AnsibleOptionsError( - "- None of the provided paths were usable. Please specify a valid path with --{0}s-path".format(context.CLIARGS['type']) - ) + raise AnsibleOptionsError("- None of the provided paths were usable. Please specify a valid path with --{0}s-path".format(context.CLIARGS['type'])) return 0 @@ -1648,65 +1594,100 @@ class GalaxyCLI(CLI): artifacts_manager.require_build_metadata = False output_format = context.CLIARGS['output_format'] + collections_search_paths = set(context.CLIARGS['collections_path']) collection_name = context.CLIARGS['collection'] - default_collections_path = set(C.COLLECTIONS_PATHS) - collections_search_paths = ( - set(context.CLIARGS['collections_path'] or []) | default_collections_path | set(AnsibleCollectionConfig.collection_paths) - ) + default_collections_path = AnsibleCollectionConfig.collection_paths collections_in_paths = {} warnings = [] path_found = False collection_found = False - - namespace_filter = None - collection_filter = None - if collection_name: - # list a specific collection - - validate_collection_name(collection_name) - namespace_filter, collection_filter = collection_name.split('.') - - collections = list(find_existing_collections( - list(collections_search_paths), - artifacts_manager, - namespace_filter=namespace_filter, - collection_filter=collection_filter, - dedupe=False - )) - - seen = set() - fqcn_width, version_width = _get_collection_widths(collections) - for collection in sorted(collections, key=lambda c: c.src): - collection_found = True - collection_path = pathlib.Path(to_text(collection.src)).parent.parent.as_posix() - - if output_format in {'yaml', 'json'}: - collections_in_paths.setdefault(collection_path, {}) - collections_in_paths[collection_path][collection.fqcn] = {'version': collection.ver} - else: - if collection_path not in seen: - _display_header( - collection_path, - 'Collection', - 'Version', - fqcn_width, - version_width - ) - seen.add(collection_path) - _display_collection(collection, fqcn_width, version_width) - - path_found = False for path in collections_search_paths: + collection_path = GalaxyCLI._resolve_path(path) if not os.path.exists(path): if path in default_collections_path: # don't warn for missing default paths continue - warnings.append("- the configured path {0} does not exist.".format(path)) - elif os.path.exists(path) and not os.path.isdir(path): - warnings.append("- the configured path {0}, exists, but it is not a directory.".format(path)) + warnings.append("- the configured path {0} does not exist.".format(collection_path)) + continue + + if not os.path.isdir(collection_path): + warnings.append("- the configured path {0}, exists, but it is not a directory.".format(collection_path)) + continue + + path_found = True + + if collection_name: + # list a specific collection + + validate_collection_name(collection_name) + namespace, collection = collection_name.split('.') + + collection_path = validate_collection_path(collection_path) + b_collection_path = to_bytes(os.path.join(collection_path, namespace, collection), errors='surrogate_or_strict') + + if not os.path.exists(b_collection_path): + warnings.append("- unable to find {0} in collection paths".format(collection_name)) + continue + + if not os.path.isdir(collection_path): + warnings.append("- the configured path {0}, exists, but it is not a directory.".format(collection_path)) + continue + + collection_found = True + + try: + collection = Requirement.from_dir_path_as_unknown( + b_collection_path, + artifacts_manager, + ) + except ValueError as val_err: + six.raise_from(AnsibleError(val_err), val_err) + + if output_format in {'yaml', 'json'}: + collections_in_paths[collection_path] = { + collection.fqcn: {'version': collection.ver} + } + + continue + + fqcn_width, version_width = _get_collection_widths([collection]) + + _display_header(collection_path, 'Collection', 'Version', fqcn_width, version_width) + _display_collection(collection, fqcn_width, version_width) + else: - path_found = True + # list all collections + collection_path = validate_collection_path(path) + if os.path.isdir(collection_path): + display.vvv("Searching {0} for collections".format(collection_path)) + collections = list(find_existing_collections( + collection_path, artifacts_manager, + )) + else: + # There was no 'ansible_collections/' directory in the path, so there + # or no collections here. + display.vvv("No 'ansible_collections' directory found at {0}".format(collection_path)) + continue + + if not collections: + display.vvv("No collections found at {0}".format(collection_path)) + continue + + if output_format in {'yaml', 'json'}: + collections_in_paths[collection_path] = { + collection.fqcn: {'version': collection.ver} for collection in collections + } + + continue + + # Display header + fqcn_width, version_width = _get_collection_widths(collections) + _display_header(collection_path, 'Collection', 'Version', fqcn_width, version_width) + + # Sort collections by the namespace and name + for collection in sorted(collections, key=to_text): + _display_collection(collection, fqcn_width, version_width) # Do not warn if the specific collection was found in any of the search paths if collection_found and collection_name: @@ -1715,10 +1696,8 @@ class GalaxyCLI(CLI): for w in warnings: display.warning(w) - if not collections and not path_found: - raise AnsibleOptionsError( - "- None of the provided paths were usable. Please specify a valid path with --{0}s-path".format(context.CLIARGS['type']) - ) + if not path_found: + raise AnsibleOptionsError("- None of the provided paths were usable. Please specify a valid path with --{0}s-path".format(context.CLIARGS['type'])) if output_format == 'json': display.display(json.dumps(collections_in_paths)) @@ -1752,8 +1731,8 @@ class GalaxyCLI(CLI): tags=context.CLIARGS['galaxy_tags'], author=context.CLIARGS['author'], page_size=page_size) if response['count'] == 0: - display.warning("No roles match your search.") - return 0 + display.display("No roles match your search.", color=C.COLOR_ERROR) + return 1 data = [u''] @@ -1792,7 +1771,6 @@ class GalaxyCLI(CLI): github_user = to_text(context.CLIARGS['github_user'], errors='surrogate_or_strict') github_repo = to_text(context.CLIARGS['github_repo'], errors='surrogate_or_strict') - rc = 0 if context.CLIARGS['check_status']: task = self.api.get_import_task(github_user=github_user, github_repo=github_repo) else: @@ -1810,7 +1788,7 @@ class GalaxyCLI(CLI): display.display('%s.%s' % (t['summary_fields']['role']['namespace'], t['summary_fields']['role']['name']), color=C.COLOR_CHANGED) display.display(u'\nTo properly namespace this role, remove each of the above and re-import %s/%s from scratch' % (github_user, github_repo), color=C.COLOR_CHANGED) - return rc + return 0 # found a single role as expected display.display("Successfully submitted import request %d" % task[0]['id']) if not context.CLIARGS['wait']: @@ -1827,13 +1805,12 @@ class GalaxyCLI(CLI): if msg['id'] not in msg_list: display.display(msg['message_text'], color=colors[msg['message_type']]) msg_list.append(msg['id']) - if (state := task[0]['state']) in ['SUCCESS', 'FAILED']: - rc = ['SUCCESS', 'FAILED'].index(state) + if task[0]['state'] in ['SUCCESS', 'FAILED']: finished = True else: time.sleep(10) - return rc + return 0 def execute_setup(self): """ Setup an integration from Github or Travis for Ansible Galaxy roles""" |