Update upstream source from recreated 2.16.5 importdebian/failed-recreation-attempt

1 files changed, 117 insertions, 108 deletions

diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py
index 67be9240..19ca0aaf 100644
--- a/lib/ansible/module_utils/basic.py
+++ b/lib/ansible/module_utils/basic.py
@@ -5,28 +5,20 @@
 from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
-FILE_ATTRIBUTES = {
-    'A': 'noatime',
-    'a': 'append',
-    'c': 'compressed',
-    'C': 'nocow',
-    'd': 'nodump',
-    'D': 'dirsync',
-    'e': 'extents',
-    'E': 'encrypted',
-    'h': 'blocksize',
-    'i': 'immutable',
-    'I': 'indexed',
-    'j': 'journalled',
-    'N': 'inline',
-    's': 'zero',
-    'S': 'synchronous',
-    't': 'notail',
-    'T': 'blockroot',
-    'u': 'undelete',
-    'X': 'compressedraw',
-    'Z': 'compresseddirty',
-}
+import sys
+
+# Used for determining if the system is running a new enough python version
+# and should only restrict on our documented minimum versions
+_PY3_MIN = sys.version_info >= (3, 6)
+_PY2_MIN = (2, 7) <= sys.version_info < (3,)
+_PY_MIN = _PY3_MIN or _PY2_MIN
+
+if not _PY_MIN:
+    print(
+        '\n{"failed": true, '
+        '"msg": "ansible-core requires a minimum of Python2 version 2.7 or Python3 version 3.6. Current version: %s"}' % ''.join(sys.version.splitlines())
+    )
+    sys.exit(1)
 
 # Ansible modules can be written in any language.
 # The functions available here can be used to do many common tasks,
@@ -49,7 +41,6 @@ import shutil
 import signal
 import stat
 import subprocess
-import sys
 import tempfile
 import time
 import traceback
@@ -101,43 +92,49 @@ from ansible.module_utils.common.text.formatters import (
     SIZE_RANGES,
 )
 
+import hashlib
+
+
+def _get_available_hash_algorithms():
+    """Return a dictionary of available hash function names and their associated function."""
+    try:
+        # Algorithms available in Python 2.7.9+ and Python 3.2+
+        # https://docs.python.org/2.7/library/hashlib.html#hashlib.algorithms_available
+        # https://docs.python.org/3.2/library/hashlib.html#hashlib.algorithms_available
+        algorithm_names = hashlib.algorithms_available
+    except AttributeError:
+        # Algorithms in Python 2.7.x (used only for Python 2.7.0 through 2.7.8)
+        # https://docs.python.org/2.7/library/hashlib.html#hashlib.hashlib.algorithms
+        algorithm_names = set(hashlib.algorithms)
+
+    algorithms = {}
+
+    for algorithm_name in algorithm_names:
+        algorithm_func = getattr(hashlib, algorithm_name, None)
+
+        if algorithm_func:
+            try:
+                # Make sure the algorithm is actually available for use.
+                # Not all algorithms listed as available are actually usable.
+                # For example, md5 is not available in FIPS mode.
+                algorithm_func()
+            except Exception:
+                pass
+            else:
+                algorithms[algorithm_name] = algorithm_func
+
+    return algorithms
+
+
+AVAILABLE_HASH_ALGORITHMS = _get_available_hash_algorithms()
+
 try:
     from ansible.module_utils.common._json_compat import json
 except ImportError as e:
     print('\n{{"msg": "Error: ansible requires the stdlib json: {0}", "failed": true}}'.format(to_native(e)))
     sys.exit(1)
 
-
-AVAILABLE_HASH_ALGORITHMS = dict()
-try:
-    import hashlib
-
-    # python 2.7.9+ and 2.7.0+
-    for attribute in ('available_algorithms', 'algorithms'):
-        algorithms = getattr(hashlib, attribute, None)
-        if algorithms:
-            break
-    if algorithms is None:
-        # python 2.5+
-        algorithms = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512')
-    for algorithm in algorithms:
-        AVAILABLE_HASH_ALGORITHMS[algorithm] = getattr(hashlib, algorithm)
-
-    # we may have been able to import md5 but it could still not be available
-    try:
-        hashlib.md5()
-    except ValueError:
-        AVAILABLE_HASH_ALGORITHMS.pop('md5', None)
-except Exception:
-    import sha
-    AVAILABLE_HASH_ALGORITHMS = {'sha1': sha.sha}
-    try:
-        import md5
-        AVAILABLE_HASH_ALGORITHMS['md5'] = md5.md5
-    except Exception:
-        pass
-
-from ansible.module_utils.common._collections_compat import (
+from ansible.module_utils.six.moves.collections_abc import (
     KeysView,
     Mapping, MutableMapping,
     Sequence, MutableSequence,
@@ -152,6 +149,7 @@ from ansible.module_utils.common.file import (
     is_executable,
     format_attributes,
     get_flags_from_attributes,
+    FILE_ATTRIBUTES,
 )
 from ansible.module_utils.common.sys_info import (
     get_distribution,
@@ -203,14 +201,14 @@ imap = map
 
 try:
     # Python 2
-    unicode  # type: ignore[has-type]  # pylint: disable=used-before-assignment
+    unicode  # type: ignore[used-before-def]  # pylint: disable=used-before-assignment
 except NameError:
     # Python 3
     unicode = text_type
 
 try:
     # Python 2
-    basestring  # type: ignore[has-type]  # pylint: disable=used-before-assignment
+    basestring  # type: ignore[used-before-def,has-type]  # pylint: disable=used-before-assignment
 except NameError:
     # Python 3
     basestring = string_types
@@ -245,20 +243,8 @@ PASSWD_ARG_RE = re.compile(r'^[-]{0,2}pass[-]?(word|wd)?')
 
 # Used for parsing symbolic file perms
 MODE_OPERATOR_RE = re.compile(r'[+=-]')
-USERS_RE = re.compile(r'[^ugo]')
-PERMS_RE = re.compile(r'[^rwxXstugo]')
-
-# Used for determining if the system is running a new enough python version
-# and should only restrict on our documented minimum versions
-_PY3_MIN = sys.version_info >= (3, 5)
-_PY2_MIN = (2, 7) <= sys.version_info < (3,)
-_PY_MIN = _PY3_MIN or _PY2_MIN
-if not _PY_MIN:
-    print(
-        '\n{"failed": true, '
-        '"msg": "ansible-core requires a minimum of Python2 version 2.7 or Python3 version 3.5. Current version: %s"}' % ''.join(sys.version.splitlines())
-    )
-    sys.exit(1)
+USERS_RE = re.compile(r'^[ugo]+$')
+PERMS_RE = re.compile(r'^[rwxXstugo]*$')
 
 
 #
@@ -1055,18 +1041,18 @@ class AnsibleModule(object):
 
             # Check if there are illegal characters in the user list
             # They can end up in 'users' because they are not split
-            if USERS_RE.match(users):
+            if not USERS_RE.match(users):
                 raise ValueError("bad symbolic permission for mode: %s" % mode)
 
             # Now we have two list of equal length, one contains the requested
             # permissions and one with the corresponding operators.
             for idx, perms in enumerate(permlist):
                 # Check if there are illegal characters in the permissions
-                if PERMS_RE.match(perms):
+                if not PERMS_RE.match(perms):
                     raise ValueError("bad symbolic permission for mode: %s" % mode)
 
                 for user in users:
-                    mode_to_apply = cls._get_octal_mode_from_symbolic_perms(path_stat, user, perms, use_umask)
+                    mode_to_apply = cls._get_octal_mode_from_symbolic_perms(path_stat, user, perms, use_umask, new_mode)
                     new_mode = cls._apply_operation_to_mode(user, opers[idx], mode_to_apply, new_mode)
 
         return new_mode
@@ -1091,9 +1077,9 @@ class AnsibleModule(object):
         return new_mode
 
     @staticmethod
-    def _get_octal_mode_from_symbolic_perms(path_stat, user, perms, use_umask):
-        prev_mode = stat.S_IMODE(path_stat.st_mode)
-
+    def _get_octal_mode_from_symbolic_perms(path_stat, user, perms, use_umask, prev_mode=None):
+        if prev_mode is None:
+            prev_mode = stat.S_IMODE(path_stat.st_mode)
         is_directory = stat.S_ISDIR(path_stat.st_mode)
         has_x_permissions = (prev_mode & EXEC_PERM_BITS) > 0
         apply_X_permission = is_directory or has_x_permissions
@@ -1503,7 +1489,19 @@ class AnsibleModule(object):
         if deprecations:
             kwargs['deprecations'] = deprecations
 
+        # preserve bools/none from no_log
+        # TODO: once python version on target high enough, dict comprh
+        preserved = {}
+        for k, v in kwargs.items():
+            if v is None or isinstance(v, bool):
+                preserved[k] = v
+
+        # strip no_log collisions
         kwargs = remove_values(kwargs, self.no_log_values)
+
+        # return preserved
+        kwargs.update(preserved)
+
         print('\n%s' % self.jsonify(kwargs))
 
     def exit_json(self, **kwargs):
@@ -1707,14 +1705,6 @@ class AnsibleModule(object):
                     tmp_dest_fd, tmp_dest_name = tempfile.mkstemp(prefix=b'.ansible_tmp', dir=b_dest_dir, suffix=b_suffix)
                 except (OSError, IOError) as e:
                     error_msg = 'The destination directory (%s) is not writable by the current user. Error was: %s' % (os.path.dirname(dest), to_native(e))
-                except TypeError:
-                    # We expect that this is happening because python3.4.x and
-                    # below can't handle byte strings in mkstemp().
-                    # Traceback would end in something like:
-                    #     file = _os.path.join(dir, pre + name + suf)
-                    # TypeError: can't concat bytes to str
-                    error_msg = ('Failed creating tmp file for atomic move.  This usually happens when using Python3 less than Python3.5. '
-                                 'Please use Python2.x or Python3.5 or greater.')
                 finally:
                     if error_msg:
                         if unsafe_writes:
@@ -1844,6 +1834,14 @@ class AnsibleModule(object):
         '''
         Execute a command, returns rc, stdout, and stderr.
 
+        The mechanism of this method for reading stdout and stderr differs from
+        that of CPython subprocess.Popen.communicate, in that this method will
+        stop reading once the spawned command has exited and stdout and stderr
+        have been consumed, as opposed to waiting until stdout/stderr are
+        closed. This can be an important distinction, when taken into account
+        that a forked or backgrounded process may hold stdout or stderr open
+        for longer than the spawned command.
+
         :arg args: is the command to run
             * If args is a list, the command will be run with shell=False.
             * If args is a string and use_unsafe_shell=False it will split args to a list and run with shell=False
@@ -2023,53 +2021,64 @@ class AnsibleModule(object):
             if before_communicate_callback:
                 before_communicate_callback(cmd)
 
-            # the communication logic here is essentially taken from that
-            # of the _communicate() function in ssh.py
-
             stdout = b''
             stderr = b''
-            try:
-                selector = selectors.DefaultSelector()
-            except (IOError, OSError):
-                # Failed to detect default selector for the given platform
-                # Select PollSelector which is supported by major platforms
+
+            # Mirror the CPython subprocess logic and preference for the selector to use.
+            # poll/select have the advantage of not requiring any extra file
+            # descriptor, contrarily to epoll/kqueue (also, they require a single
+            # syscall).
+            if hasattr(selectors, 'PollSelector'):
                 selector = selectors.PollSelector()
+            else:
+                selector = selectors.SelectSelector()
+
+            if data:
+                if not binary_data:
+                    data += '\n'
+                if isinstance(data, text_type):
+                    data = to_bytes(data)
 
             selector.register(cmd.stdout, selectors.EVENT_READ)
             selector.register(cmd.stderr, selectors.EVENT_READ)
+
             if os.name == 'posix':
                 fcntl.fcntl(cmd.stdout.fileno(), fcntl.F_SETFL, fcntl.fcntl(cmd.stdout.fileno(), fcntl.F_GETFL) | os.O_NONBLOCK)
                 fcntl.fcntl(cmd.stderr.fileno(), fcntl.F_SETFL, fcntl.fcntl(cmd.stderr.fileno(), fcntl.F_GETFL) | os.O_NONBLOCK)
 
             if data:
-                if not binary_data:
-                    data += '\n'
-                if isinstance(data, text_type):
-                    data = to_bytes(data)
                 cmd.stdin.write(data)
                 cmd.stdin.close()
 
             while True:
+                # A timeout of 1 is both a little short and a little long.
+                # With None we could deadlock, with a lower value we would
+                # waste cycles. As it is, this is a mild inconvenience if
+                # we need to exit, and likely doesn't waste too many cycles
                 events = selector.select(1)
+                stdout_changed = False
                 for key, event in events:
-                    b_chunk = key.fileobj.read()
-                    if b_chunk == b(''):
+                    b_chunk = key.fileobj.read(32768)
+                    if not b_chunk:
                         selector.unregister(key.fileobj)
-                    if key.fileobj == cmd.stdout:
+                    elif key.fileobj == cmd.stdout:
                         stdout += b_chunk
+                        stdout_changed = True
                     elif key.fileobj == cmd.stderr:
                         stderr += b_chunk
-                # if we're checking for prompts, do it now
-                if prompt_re:
-                    if prompt_re.search(stdout) and not data:
-                        if encoding:
-                            stdout = to_native(stdout, encoding=encoding, errors=errors)
-                        return (257, stdout, "A prompt was encountered while running a command, but no input data was specified")
-                # only break out if no pipes are left to read or
-                # the pipes are completely read and
-                # the process is terminated
+
+                # if we're checking for prompts, do it now, but only if stdout
+                # actually changed since the last loop
+                if prompt_re and stdout_changed and prompt_re.search(stdout) and not data:
+                    if encoding:
+                        stdout = to_native(stdout, encoding=encoding, errors=errors)
+                    return (257, stdout, "A prompt was encountered while running a command, but no input data was specified")
+
+                # break out if no pipes are left to read or the pipes are completely read
+                # and the process is terminated
                 if (not events or not selector.get_map()) and cmd.poll() is not None:
                     break
+
                 # No pipes are left to read but process is not yet terminated
                 # Only then it is safe to wait for the process to be finished
                 # NOTE: Actually cmd.poll() is always None here if no selectors are left