diff options
Diffstat (limited to 'lib/ansible/module_utils/basic.py')
| -rw-r--r-- | lib/ansible/module_utils/basic.py | 225 |
1 files changed, 117 insertions, 108 deletions
diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index 67be9240..19ca0aaf 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -5,28 +5,20 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type -FILE_ATTRIBUTES = { - 'A': 'noatime', - 'a': 'append', - 'c': 'compressed', - 'C': 'nocow', - 'd': 'nodump', - 'D': 'dirsync', - 'e': 'extents', - 'E': 'encrypted', - 'h': 'blocksize', - 'i': 'immutable', - 'I': 'indexed', - 'j': 'journalled', - 'N': 'inline', - 's': 'zero', - 'S': 'synchronous', - 't': 'notail', - 'T': 'blockroot', - 'u': 'undelete', - 'X': 'compressedraw', - 'Z': 'compresseddirty', -} +import sys + +# Used for determining if the system is running a new enough python version +# and should only restrict on our documented minimum versions +_PY3_MIN = sys.version_info >= (3, 6) +_PY2_MIN = (2, 7) <= sys.version_info < (3,) +_PY_MIN = _PY3_MIN or _PY2_MIN + +if not _PY_MIN: + print( + '\n{"failed": true, ' + '"msg": "ansible-core requires a minimum of Python2 version 2.7 or Python3 version 3.6. Current version: %s"}' % ''.join(sys.version.splitlines()) + ) + sys.exit(1) # Ansible modules can be written in any language. # The functions available here can be used to do many common tasks, @@ -49,7 +41,6 @@ import shutil import signal import stat import subprocess -import sys import tempfile import time import traceback @@ -101,43 +92,49 @@ from ansible.module_utils.common.text.formatters import ( SIZE_RANGES, ) +import hashlib + + +def _get_available_hash_algorithms(): + """Return a dictionary of available hash function names and their associated function.""" + try: + # Algorithms available in Python 2.7.9+ and Python 3.2+ + # https://docs.python.org/2.7/library/hashlib.html#hashlib.algorithms_available + # https://docs.python.org/3.2/library/hashlib.html#hashlib.algorithms_available + algorithm_names = hashlib.algorithms_available + except AttributeError: + # Algorithms in Python 2.7.x (used only for Python 2.7.0 through 2.7.8) + # https://docs.python.org/2.7/library/hashlib.html#hashlib.hashlib.algorithms + algorithm_names = set(hashlib.algorithms) + + algorithms = {} + + for algorithm_name in algorithm_names: + algorithm_func = getattr(hashlib, algorithm_name, None) + + if algorithm_func: + try: + # Make sure the algorithm is actually available for use. + # Not all algorithms listed as available are actually usable. + # For example, md5 is not available in FIPS mode. + algorithm_func() + except Exception: + pass + else: + algorithms[algorithm_name] = algorithm_func + + return algorithms + + +AVAILABLE_HASH_ALGORITHMS = _get_available_hash_algorithms() + try: from ansible.module_utils.common._json_compat import json except ImportError as e: print('\n{{"msg": "Error: ansible requires the stdlib json: {0}", "failed": true}}'.format(to_native(e))) sys.exit(1) - -AVAILABLE_HASH_ALGORITHMS = dict() -try: - import hashlib - - # python 2.7.9+ and 2.7.0+ - for attribute in ('available_algorithms', 'algorithms'): - algorithms = getattr(hashlib, attribute, None) - if algorithms: - break - if algorithms is None: - # python 2.5+ - algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512') - for algorithm in algorithms: - AVAILABLE_HASH_ALGORITHMS[algorithm] = getattr(hashlib, algorithm) - - # we may have been able to import md5 but it could still not be available - try: - hashlib.md5() - except ValueError: - AVAILABLE_HASH_ALGORITHMS.pop('md5', None) -except Exception: - import sha - AVAILABLE_HASH_ALGORITHMS = {'sha1': sha.sha} - try: - import md5 - AVAILABLE_HASH_ALGORITHMS['md5'] = md5.md5 - except Exception: - pass - -from ansible.module_utils.common._collections_compat import ( +from ansible.module_utils.six.moves.collections_abc import ( KeysView, Mapping, MutableMapping, Sequence, MutableSequence, @@ -152,6 +149,7 @@ from ansible.module_utils.common.file import ( is_executable, format_attributes, get_flags_from_attributes, + FILE_ATTRIBUTES, ) from ansible.module_utils.common.sys_info import ( get_distribution, @@ -203,14 +201,14 @@ imap = map try: # Python 2 - unicode # type: ignore[has-type] # pylint: disable=used-before-assignment + unicode # type: ignore[used-before-def] # pylint: disable=used-before-assignment except NameError: # Python 3 unicode = text_type try: # Python 2 - basestring # type: ignore[has-type] # pylint: disable=used-before-assignment + basestring # type: ignore[used-before-def,has-type] # pylint: disable=used-before-assignment except NameError: # Python 3 basestring = string_types @@ -245,20 +243,8 @@ PASSWD_ARG_RE = re.compile(r'^[-]{0,2}pass[-]?(word|wd)?') # Used for parsing symbolic file perms MODE_OPERATOR_RE = re.compile(r'[+=-]') -USERS_RE = re.compile(r'[^ugo]') -PERMS_RE = re.compile(r'[^rwxXstugo]') - -# Used for determining if the system is running a new enough python version -# and should only restrict on our documented minimum versions -_PY3_MIN = sys.version_info >= (3, 5) -_PY2_MIN = (2, 7) <= sys.version_info < (3,) -_PY_MIN = _PY3_MIN or _PY2_MIN -if not _PY_MIN: - print( - '\n{"failed": true, ' - '"msg": "ansible-core requires a minimum of Python2 version 2.7 or Python3 version 3.5. Current version: %s"}' % ''.join(sys.version.splitlines()) - ) - sys.exit(1) +USERS_RE = re.compile(r'^[ugo]+$') +PERMS_RE = re.compile(r'^[rwxXstugo]*$') # @@ -1055,18 +1041,18 @@ class AnsibleModule(object): # Check if there are illegal characters in the user list # They can end up in 'users' because they are not split - if USERS_RE.match(users): + if not USERS_RE.match(users): raise ValueError("bad symbolic permission for mode: %s" % mode) # Now we have two list of equal length, one contains the requested # permissions and one with the corresponding operators. for idx, perms in enumerate(permlist): # Check if there are illegal characters in the permissions - if PERMS_RE.match(perms): + if not PERMS_RE.match(perms): raise ValueError("bad symbolic permission for mode: %s" % mode) for user in users: - mode_to_apply = cls._get_octal_mode_from_symbolic_perms(path_stat, user, perms, use_umask) + mode_to_apply = cls._get_octal_mode_from_symbolic_perms(path_stat, user, perms, use_umask, new_mode) new_mode = cls._apply_operation_to_mode(user, opers[idx], mode_to_apply, new_mode) return new_mode @@ -1091,9 +1077,9 @@ class AnsibleModule(object): return new_mode @staticmethod - def _get_octal_mode_from_symbolic_perms(path_stat, user, perms, use_umask): - prev_mode = stat.S_IMODE(path_stat.st_mode) - + def _get_octal_mode_from_symbolic_perms(path_stat, user, perms, use_umask, prev_mode=None): + if prev_mode is None: + prev_mode = stat.S_IMODE(path_stat.st_mode) is_directory = stat.S_ISDIR(path_stat.st_mode) has_x_permissions = (prev_mode & EXEC_PERM_BITS) > 0 apply_X_permission = is_directory or has_x_permissions @@ -1503,7 +1489,19 @@ class AnsibleModule(object): if deprecations: kwargs['deprecations'] = deprecations + # preserve bools/none from no_log + # TODO: once python version on target high enough, dict comprh + preserved = {} + for k, v in kwargs.items(): + if v is None or isinstance(v, bool): + preserved[k] = v + + # strip no_log collisions kwargs = remove_values(kwargs, self.no_log_values) + + # return preserved + kwargs.update(preserved) + print('\n%s' % self.jsonify(kwargs)) def exit_json(self, **kwargs): @@ -1707,14 +1705,6 @@ class AnsibleModule(object): tmp_dest_fd, tmp_dest_name = tempfile.mkstemp(prefix=b'.ansible_tmp', dir=b_dest_dir, suffix=b_suffix) except (OSError, IOError) as e: error_msg = 'The destination directory (%s) is not writable by the current user. Error was: %s' % (os.path.dirname(dest), to_native(e)) - except TypeError: - # We expect that this is happening because python3.4.x and - # below can't handle byte strings in mkstemp(). - # Traceback would end in something like: - # file = _os.path.join(dir, pre + name + suf) - # TypeError: can't concat bytes to str - error_msg = ('Failed creating tmp file for atomic move. This usually happens when using Python3 less than Python3.5. ' - 'Please use Python2.x or Python3.5 or greater.') finally: if error_msg: if unsafe_writes: @@ -1844,6 +1834,14 @@ class AnsibleModule(object): ''' Execute a command, returns rc, stdout, and stderr. + The mechanism of this method for reading stdout and stderr differs from + that of CPython subprocess.Popen.communicate, in that this method will + stop reading once the spawned command has exited and stdout and stderr + have been consumed, as opposed to waiting until stdout/stderr are + closed. This can be an important distinction, when taken into account + that a forked or backgrounded process may hold stdout or stderr open + for longer than the spawned command. + :arg args: is the command to run * If args is a list, the command will be run with shell=False. * If args is a string and use_unsafe_shell=False it will split args to a list and run with shell=False @@ -2023,53 +2021,64 @@ class AnsibleModule(object): if before_communicate_callback: before_communicate_callback(cmd) - # the communication logic here is essentially taken from that - # of the _communicate() function in ssh.py - stdout = b'' stderr = b'' - try: - selector = selectors.DefaultSelector() - except (IOError, OSError): - # Failed to detect default selector for the given platform - # Select PollSelector which is supported by major platforms + + # Mirror the CPython subprocess logic and preference for the selector to use. + # poll/select have the advantage of not requiring any extra file + # descriptor, contrarily to epoll/kqueue (also, they require a single + # syscall). + if hasattr(selectors, 'PollSelector'): selector = selectors.PollSelector() + else: + selector = selectors.SelectSelector() + + if data: + if not binary_data: + data += '\n' + if isinstance(data, text_type): + data = to_bytes(data) selector.register(cmd.stdout, selectors.EVENT_READ) selector.register(cmd.stderr, selectors.EVENT_READ) + if os.name == 'posix': fcntl.fcntl(cmd.stdout.fileno(), fcntl.F_SETFL, fcntl.fcntl(cmd.stdout.fileno(), fcntl.F_GETFL) | os.O_NONBLOCK) fcntl.fcntl(cmd.stderr.fileno(), fcntl.F_SETFL, fcntl.fcntl(cmd.stderr.fileno(), fcntl.F_GETFL) | os.O_NONBLOCK) if data: - if not binary_data: - data += '\n' - if isinstance(data, text_type): - data = to_bytes(data) cmd.stdin.write(data) cmd.stdin.close() while True: + # A timeout of 1 is both a little short and a little long. + # With None we could deadlock, with a lower value we would + # waste cycles. As it is, this is a mild inconvenience if + # we need to exit, and likely doesn't waste too many cycles events = selector.select(1) + stdout_changed = False for key, event in events: - b_chunk = key.fileobj.read() - if b_chunk == b(''): + b_chunk = key.fileobj.read(32768) + if not b_chunk: selector.unregister(key.fileobj) - if key.fileobj == cmd.stdout: + elif key.fileobj == cmd.stdout: stdout += b_chunk + stdout_changed = True elif key.fileobj == cmd.stderr: stderr += b_chunk - # if we're checking for prompts, do it now - if prompt_re: - if prompt_re.search(stdout) and not data: - if encoding: - stdout = to_native(stdout, encoding=encoding, errors=errors) - return (257, stdout, "A prompt was encountered while running a command, but no input data was specified") - # only break out if no pipes are left to read or - # the pipes are completely read and - # the process is terminated + + # if we're checking for prompts, do it now, but only if stdout + # actually changed since the last loop + if prompt_re and stdout_changed and prompt_re.search(stdout) and not data: + if encoding: + stdout = to_native(stdout, encoding=encoding, errors=errors) + return (257, stdout, "A prompt was encountered while running a command, but no input data was specified") + + # break out if no pipes are left to read or the pipes are completely read + # and the process is terminated if (not events or not selector.get_map()) and cmd.poll() is not None: break + # No pipes are left to read but process is not yet terminated # Only then it is safe to wait for the process to be finished # NOTE: Actually cmd.poll() is always None here if no selectors are left |