New upstream version 2.14.0

8 files changed, 173 insertions, 116 deletions

diff --git a/lib/ansible/utils/collection_loader/_collection_finder.py b/lib/ansible/utils/collection_loader/_collection_finder.py
index c581abf2..d3a8765c 100644
--- a/lib/ansible/utils/collection_loader/_collection_finder.py
+++ b/lib/ansible/utils/collection_loader/_collection_finder.py
@@ -914,7 +914,7 @@ class AnsibleCollectionRef:
         """
         legacy_plugin_dir_name = to_text(legacy_plugin_dir_name)
 
-        plugin_type = legacy_plugin_dir_name.replace(u'_plugins', u'')
+        plugin_type = legacy_plugin_dir_name.removesuffix(u'_plugins')
 
         if plugin_type == u'library':
             plugin_type = u'modules'
@@ -960,6 +960,18 @@ class AnsibleCollectionRef:
         )
 
 
+def _get_collection_path(collection_name):
+    collection_name = to_native(collection_name)
+    if not collection_name or not isinstance(collection_name, string_types) or len(collection_name.split('.')) != 2:
+        raise ValueError('collection_name must be a non-empty string of the form namespace.collection')
+    try:
+        collection_pkg = import_module('ansible_collections.' + collection_name)
+    except ImportError:
+        raise ValueError('unable to locate collection {0}'.format(collection_name))
+
+    return to_native(os.path.dirname(to_bytes(collection_pkg.__file__)))
+
+
 def _get_collection_playbook_path(playbook):
 
     acr = AnsibleCollectionRef.try_parse_fqcr(playbook, u'playbook')
diff --git a/lib/ansible/utils/display.py b/lib/ansible/utils/display.py
index b9d24654..c3a5de98 100644
--- a/lib/ansible/utils/display.py
+++ b/lib/ansible/utils/display.py
@@ -19,16 +19,15 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import ctypes.util
-import errno
 import fcntl
 import getpass
-import locale
 import logging
 import os
 import random
 import subprocess
 import sys
 import textwrap
+import threading
 import time
 
 from struct import unpack, pack
@@ -39,6 +38,7 @@ from ansible.errors import AnsibleError, AnsibleAssertionError
 from ansible.module_utils._text import to_bytes, to_text
 from ansible.module_utils.six import text_type
 from ansible.utils.color import stringc
+from ansible.utils.multiprocessing import context as multiprocessing_context
 from ansible.utils.singleton import Singleton
 from ansible.utils.unsafe_proxy import wrap_var
 
@@ -51,24 +51,6 @@ _LIBC.wcswidth.argtypes = (ctypes.c_wchar_p, ctypes.c_int)
 # Max for c_int
 _MAX_INT = 2 ** (ctypes.sizeof(ctypes.c_int) * 8 - 1) - 1
 
-_LOCALE_INITIALIZED = False
-_LOCALE_INITIALIZATION_ERR = None
-
-
-def initialize_locale():
-    """Set the locale to the users default setting
-    and set ``_LOCALE_INITIALIZED`` to indicate whether
-    ``get_text_width`` may run into trouble
-    """
-    global _LOCALE_INITIALIZED, _LOCALE_INITIALIZATION_ERR
-    if _LOCALE_INITIALIZED is False:
-        try:
-            locale.setlocale(locale.LC_ALL, '')
-        except locale.Error as e:
-            _LOCALE_INITIALIZATION_ERR = e
-        else:
-            _LOCALE_INITIALIZED = True
-
 
 def get_text_width(text):
     """Function that utilizes ``wcswidth`` or ``wcwidth`` to determine the
@@ -76,27 +58,11 @@ def get_text_width(text):
 
     We try first with ``wcswidth``, and fallback to iterating each
     character and using wcwidth individually, falling back to a value of 0
-    for non-printable wide characters
-
-    On Py2, this depends on ``locale.setlocale(locale.LC_ALL, '')``,
-    that in the case of Ansible is done in ``bin/ansible``
+    for non-printable wide characters.
     """
     if not isinstance(text, text_type):
         raise TypeError('get_text_width requires text, not %s' % type(text))
 
-    if _LOCALE_INITIALIZATION_ERR:
-        Display().warning(
-            'An error occurred while calling ansible.utils.display.initialize_locale '
-            '(%s). This may result in incorrectly calculated text widths that can '
-            'cause Display to print incorrect line lengths' % _LOCALE_INITIALIZATION_ERR
-        )
-    elif not _LOCALE_INITIALIZED:
-        Display().warning(
-            'ansible.utils.display.initialize_locale has not been called, '
-            'this may result in incorrectly calculated text widths that can '
-            'cause Display to print incorrect line lengths'
-        )
-
     try:
         width = _LIBC.wcswidth(text, _MAX_INT)
     except ctypes.ArgumentError:
@@ -128,10 +94,9 @@ def get_text_width(text):
             w = 0
         width += w
 
-    if width == 0 and counter and not _LOCALE_INITIALIZED:
+    if width == 0 and counter:
         raise EnvironmentError(
-            'ansible.utils.display.initialize_locale has not been called, '
-            'and get_text_width could not calculate text width of %r' % text
+            'get_text_width could not calculate text width of %r' % text
         )
 
     # It doesn't make sense to have a negative printable width
@@ -202,6 +167,10 @@ class Display(metaclass=Singleton):
 
     def __init__(self, verbosity=0):
 
+        self._final_q = None
+
+        self._lock = threading.RLock()
+
         self.columns = None
         self.verbosity = verbosity
 
@@ -230,6 +199,16 @@ class Display(metaclass=Singleton):
 
         self._set_column_width()
 
+    def set_queue(self, queue):
+        """Set the _final_q on Display, so that we know to proxy display over the queue
+        instead of directly writing to stdout/stderr from forks
+
+        This is only needed in ansible.executor.process.worker:WorkerProcess._run
+        """
+        if multiprocessing_context.parent_process() is None:
+            raise RuntimeError('queue cannot be set in parent process')
+        self._final_q = queue
+
     def set_cowsay_info(self):
         if C.ANSIBLE_NOCOWS:
             return
@@ -247,6 +226,13 @@ class Display(metaclass=Singleton):
         Note: msg *must* be a unicode string to prevent UnicodeError tracebacks.
         """
 
+        if self._final_q:
+            # If _final_q is set, that means we are in a WorkerProcess
+            # and instead of displaying messages directly from the fork
+            # we will proxy them through the queue
+            return self._final_q.send_display(msg, color=color, stderr=stderr,
+                                              screen_only=screen_only, log_only=log_only, newline=newline)
+
         nocolor = msg
 
         if not log_only:
@@ -263,12 +249,6 @@ class Display(metaclass=Singleton):
             if has_newline or newline:
                 msg2 = msg2 + u'\n'
 
-            msg2 = to_bytes(msg2, encoding=self._output_encoding(stderr=stderr))
-            # Convert back to text string
-            # We first convert to a byte string so that we get rid of
-            # characters that are invalid in the user's locale
-            msg2 = to_text(msg2, self._output_encoding(stderr=stderr), errors='replace')
-
             # Note: After Display() class is refactored need to update the log capture
             # code in 'bin/ansible-connection' (and other relevant places).
             if not stderr:
@@ -276,23 +256,24 @@ class Display(metaclass=Singleton):
             else:
                 fileobj = sys.stderr
 
-            fileobj.write(msg2)
-
-            try:
-                fileobj.flush()
-            except IOError as e:
-                # Ignore EPIPE in case fileobj has been prematurely closed, eg.
-                # when piping to "head -n1"
-                if e.errno != errno.EPIPE:
-                    raise
+            with self._lock:
+                fileobj.write(msg2)
+
+            # With locks, and the fact that we aren't printing from forks
+            # just write, and let the system flush. Everything should come out peachy
+            # I've left this code for historical purposes, or in case we need to add this
+            # back at a later date. For now ``TaskQueueManager.cleanup`` will perform a
+            # final flush at shutdown.
+            # try:
+            #     fileobj.flush()
+            # except IOError as e:
+            #     # Ignore EPIPE in case fileobj has been prematurely closed, eg.
+            #     # when piping to "head -n1"
+            #     if e.errno != errno.EPIPE:
+            #         raise
 
         if logger and not screen_only:
-            # We first convert to a byte string so that we get rid of
-            # color and characters that are invalid in the user's locale
-            msg2 = to_bytes(nocolor.lstrip(u'\n'))
-
-            # Convert back to text string
-            msg2 = to_text(msg2, self._output_encoding(stderr=stderr))
+            msg2 = nocolor.lstrip('\n')
 
             lvl = logging.INFO
             if color:
@@ -460,15 +441,10 @@ class Display(metaclass=Singleton):
 
     @staticmethod
     def prompt(msg, private=False):
-        prompt_string = to_bytes(msg, encoding=Display._output_encoding())
-        # Convert back into text.  We do this double conversion
-        # to get rid of characters that are illegal in the user's locale
-        prompt_string = to_text(prompt_string)
-
         if private:
-            return getpass.getpass(prompt_string)
+            return getpass.getpass(msg)
         else:
-            return input(prompt_string)
+            return input(msg)
 
     def do_var_prompt(self, varname, private=True, prompt=None, encrypt=None, confirm=False, salt_size=None, salt=None, default=None, unsafe=None):
 
@@ -513,16 +489,6 @@ class Display(metaclass=Singleton):
             result = wrap_var(result)
         return result
 
-    @staticmethod
-    def _output_encoding(stderr=False):
-        encoding = locale.getpreferredencoding()
-        # https://bugs.python.org/issue6202
-        # Python2 hardcodes an obsolete value on Mac.  Use MacOSX defaults
-        # instead.
-        if encoding in ('mac-roman',):
-            encoding = 'utf-8'
-        return encoding
-
     def _set_column_width(self):
         if os.isatty(1):
             tty_size = unpack('HHHH', fcntl.ioctl(1, TIOCGWINSZ, pack('HHHH', 0, 0, 0, 0)))[1]
diff --git a/lib/ansible/utils/encrypt.py b/lib/ansible/utils/encrypt.py
index 4cb70b70..3a8642d8 100644
--- a/lib/ansible/utils/encrypt.py
+++ b/lib/ansible/utils/encrypt.py
@@ -99,6 +99,15 @@ class CryptHash(BaseHash):
 
         if algorithm not in self.algorithms:
             raise AnsibleError("crypt.crypt does not support '%s' algorithm" % self.algorithm)
+
+        display.deprecated(
+            "Encryption using the Python crypt module is deprecated. The "
+            "Python crypt module is deprecated and will be removed from "
+            "Python 3.13. Install the passlib library for continued "
+            "encryption functionality.",
+            version=2.17
+        )
+
         self.algo_data = self.algorithms[algorithm]
 
     def hash(self, secret, salt=None, salt_size=None, rounds=None, ident=None):
diff --git a/lib/ansible/utils/listify.py b/lib/ansible/utils/listify.py
index 4f2ae9d4..0e6a8724 100644
--- a/lib/ansible/utils/listify.py
+++ b/lib/ansible/utils/listify.py
@@ -22,12 +22,18 @@ __metaclass__ = type
 from collections.abc import Iterable
 
 from ansible.module_utils.six import string_types
+from ansible.utils.display import Display
 
+display = Display()
 
 __all__ = ['listify_lookup_plugin_terms']
 
 
-def listify_lookup_plugin_terms(terms, templar, loader, fail_on_undefined=True, convert_bare=False):
+def listify_lookup_plugin_terms(terms, templar, loader=None, fail_on_undefined=True, convert_bare=False):
+
+    if loader is not None:
+        display.deprecated('"listify_lookup_plugin_terms" does not use "dataloader" anymore, the ability to pass it in will be removed in future versions.',
+                           version='2.18')
 
     if isinstance(terms, string_types):
         terms = templar.template(terms.strip(), convert_bare=convert_bare, fail_on_undefined=fail_on_undefined)
diff --git a/lib/ansible/utils/path.py b/lib/ansible/utils/path.py
index df2769fb..f876addf 100644
--- a/lib/ansible/utils/path.py
+++ b/lib/ansible/utils/path.py
@@ -134,7 +134,7 @@ def cleanup_tmp_file(path, warn=False):
         pass
 
 
-def is_subpath(child, parent):
+def is_subpath(child, parent, real=False):
     """
     Compares paths to check if one is contained in the other
     :arg: child: Path to test
@@ -145,6 +145,10 @@ def is_subpath(child, parent):
     abs_child = unfrackpath(child, follow=False)
     abs_parent = unfrackpath(parent, follow=False)
 
+    if real:
+        abs_child = os.path.realpath(abs_child)
+        abs_parent = os.path.realpath(abs_parent)
+
     c = abs_child.split(os.path.sep)
     p = abs_parent.split(os.path.sep)
 
diff --git a/lib/ansible/utils/plugin_docs.py b/lib/ansible/utils/plugin_docs.py
index 3499800c..3af26789 100644
--- a/lib/ansible/utils/plugin_docs.py
+++ b/lib/ansible/utils/plugin_docs.py
@@ -5,27 +5,20 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 from collections.abc import MutableMapping, MutableSet, MutableSequence
+from pathlib import Path
 
 from ansible import constants as C
 from ansible.release import __version__ as ansible_version
-from ansible.errors import AnsibleError
+from ansible.errors import AnsibleError, AnsibleParserError, AnsiblePluginNotFound
 from ansible.module_utils.six import string_types
 from ansible.module_utils._text import to_native
 from ansible.parsing.plugin_docs import read_docstring
 from ansible.parsing.yaml.loader import AnsibleLoader
 from ansible.utils.display import Display
-from ansible.utils.vars import combine_vars
 
 display = Display()
 
 
-# modules that are ok that they do not have documentation strings
-REJECTLIST = {
-    'MODULE': frozenset(('async_wrapper',)),
-    'CACHE': frozenset(('base',)),
-}
-
-
 def merge_fragment(target, source):
 
     for key, value in source.items():
@@ -170,10 +163,8 @@ def add_fragments(doc, filename, fragment_loader, is_module=False):
 
         fragment = AnsibleLoader(fragment_yaml, file_name=filename).get_single_data()
 
-        real_collection_name = 'ansible.builtin'
-        real_fragment_name = getattr(fragment_class, '_load_name')
-        if real_fragment_name.startswith('ansible_collections.'):
-            real_collection_name = '.'.join(real_fragment_name.split('.')[1:3])
+        real_fragment_name = getattr(fragment_class, 'ansible_name')
+        real_collection_name = '.'.join(real_fragment_name.split('.')[0:2]) if '.' in real_fragment_name else ''
         add_collection_to_versions_and_dates(fragment, real_collection_name, is_module=is_module)
 
         if 'notes' in fragment:
@@ -214,11 +205,20 @@ def add_fragments(doc, filename, fragment_loader, is_module=False):
         raise AnsibleError('unknown doc_fragment(s) in file {0}: {1}'.format(filename, to_native(', '.join(unknown_fragments))))
 
 
-def get_docstring(filename, fragment_loader, verbose=False, ignore_errors=False, collection_name=None, is_module=False):
+def get_docstring(filename, fragment_loader, verbose=False, ignore_errors=False, collection_name=None, is_module=None, plugin_type=None):
     """
     DOCUMENTATION can be extended using documentation fragments loaded by the PluginLoader from the doc_fragments plugins.
     """
 
+    if is_module is None:
+        if plugin_type is None:
+            is_module = False
+        else:
+            is_module = (plugin_type == 'module')
+    else:
+        # TODO deprecate is_module argument, now that we have 'type'
+        pass
+
     data = read_docstring(filename, verbose=verbose, ignore_errors=ignore_errors)
 
     if data.get('doc', False):
@@ -269,3 +269,83 @@ def get_versioned_doclink(path):
         return '{0}{1}/{2}'.format(base_url, doc_version, path)
     except Exception as ex:
         return '(unable to create versioned doc link for path {0}: {1})'.format(path, to_native(ex))
+
+
+def _find_adjacent(path, plugin, extensions):
+
+    adjacent = Path(path)
+
+    plugin_base_name = plugin.split('.')[-1]
+    if adjacent.stem != plugin_base_name:
+        # this should only affect filters/tests
+        adjacent = adjacent.with_name(plugin_base_name)
+
+    paths = []
+    for ext in extensions:
+        candidate = adjacent.with_suffix(ext)
+        if candidate == adjacent:
+            # we're looking for an adjacent file, skip this since it's identical
+            continue
+        if candidate.exists():
+            paths.append(to_native(candidate))
+
+    return paths
+
+
+def find_plugin_docfile(plugin, plugin_type, loader):
+    '''  if the plugin lives in a non-python file (eg, win_X.ps1), require the corresponding 'sidecar' file for docs '''
+
+    context = loader.find_plugin_with_context(plugin, ignore_deprecated=False, check_aliases=True)
+    if (not context or not context.resolved) and plugin_type in ('filter', 'test'):
+        # should only happen for filters/test
+        plugin_obj, context = loader.get_with_context(plugin)
+
+    if not context or not context.resolved:
+        raise AnsiblePluginNotFound('%s was not found' % (plugin), plugin_load_context=context)
+
+    docfile = Path(context.plugin_resolved_path)
+    if docfile.suffix not in C.DOC_EXTENSIONS:
+        # only look for adjacent if plugin file does not support documents
+        filenames = _find_adjacent(docfile, plugin, C.DOC_EXTENSIONS)
+        filename = filenames[0] if filenames else None
+    else:
+        filename = to_native(docfile)
+
+    if filename is None:
+        raise AnsibleError('%s cannot contain DOCUMENTATION nor does it have a companion documentation file' % (plugin))
+
+    return filename, context.plugin_resolved_collection
+
+
+def get_plugin_docs(plugin, plugin_type, loader, fragment_loader, verbose):
+
+    docs = []
+
+    # find plugin doc file, if it doesn't exist this will throw error, we let it through
+    # can raise exception and short circuit when 'not found'
+    filename, collection_name = find_plugin_docfile(plugin, plugin_type, loader)
+
+    try:
+        docs = get_docstring(filename, fragment_loader, verbose=verbose, collection_name=collection_name, plugin_type=plugin_type)
+    except Exception as e:
+        raise AnsibleParserError('%s did not contain a DOCUMENTATION attribute (%s)' % (plugin, filename), orig_exc=e)
+
+    # no good? try adjacent
+    if not docs[0]:
+        for newfile in _find_adjacent(filename, plugin, C.DOC_EXTENSIONS):
+            try:
+                docs = get_docstring(newfile, fragment_loader, verbose=verbose, collection_name=collection_name, plugin_type=plugin_type)
+                filename = newfile
+                if docs[0] is not None:
+                    break
+            except Exception as e:
+                raise AnsibleParserError('Adjacent file %s did not contain a DOCUMENTATION attribute (%s)' % (plugin, filename), orig_exc=e)
+
+    # add extra data to docs[0] (aka 'DOCUMENTATION')
+    if docs[0] is None:
+        raise AnsibleParserError('No documentation available for %s (%s)' % (plugin, filename))
+    else:
+        docs[0]['filename'] = filename
+        docs[0]['collection'] = collection_name
+
+    return docs
diff --git a/lib/ansible/utils/unsafe_proxy.py b/lib/ansible/utils/unsafe_proxy.py
index e10fa8a0..d78ebf6e 100644
--- a/lib/ansible/utils/unsafe_proxy.py
+++ b/lib/ansible/utils/unsafe_proxy.py
@@ -84,25 +84,6 @@ class NativeJinjaUnsafeText(NativeJinjaText, AnsibleUnsafeText):
     pass
 
 
-class UnsafeProxy(object):
-    def __new__(cls, obj, *args, **kwargs):
-        from ansible.utils.display import Display
-        Display().deprecated(
-            'UnsafeProxy is being deprecated. Use wrap_var or AnsibleUnsafeBytes/AnsibleUnsafeText directly instead',
-            version='2.13', collection_name='ansible.builtin'
-        )
-        # In our usage we should only receive unicode strings.
-        # This conditional and conversion exists to sanity check the values
-        # we're given but we may want to take it out for testing and sanitize
-        # our input instead.
-        if isinstance(obj, AnsibleUnsafe):
-            return obj
-
-        if isinstance(obj, string_types):
-            obj = AnsibleUnsafeText(to_text(obj, errors='surrogate_or_strict'))
-        return obj
-
-
 def _wrap_dict(v):
     return dict((wrap_var(k), wrap_var(item)) for k, item in v.items())
 
diff --git a/lib/ansible/utils/vars.py b/lib/ansible/utils/vars.py
index 74bf1425..a3224c8b 100644
--- a/lib/ansible/utils/vars.py
+++ b/lib/ansible/utils/vars.py
@@ -88,8 +88,7 @@ def combine_vars(a, b, merge=None):
     else:
         # HASH_BEHAVIOUR == 'replace'
         _validate_mutable_mappings(a, b)
-        result = a.copy()
-        result.update(b)
+        result = a | b
         return result