Update upstream source from recreated 2.16.5 importdebian/failed-recreation-attempt

1 files changed, 114 insertions, 34 deletions

diff --git a/lib/ansible/plugins/loader.py b/lib/ansible/plugins/loader.py
index 8b7fbfce..9ff19bbb 100644
--- a/lib/ansible/plugins/loader.py
+++ b/lib/ansible/plugins/loader.py
@@ -17,6 +17,7 @@ import warnings
 from collections import defaultdict, namedtuple
 from traceback import format_exc
 
+import ansible.module_utils.compat.typing as t
 
 from .filter import AnsibleJinja2Filter
 from .test import AnsibleJinja2Test
@@ -24,7 +25,7 @@ from .test import AnsibleJinja2Test
 from ansible import __version__ as ansible_version
 from ansible import constants as C
 from ansible.errors import AnsibleError, AnsiblePluginCircularRedirect, AnsiblePluginRemovedError, AnsibleCollectionUnsupportedVersionError
-from ansible.module_utils._text import to_bytes, to_text, to_native
+from ansible.module_utils.common.text.converters import to_bytes, to_text, to_native
 from ansible.module_utils.compat.importlib import import_module
 from ansible.module_utils.six import string_types
 from ansible.parsing.utils.yaml import from_yaml
@@ -33,7 +34,8 @@ from ansible.plugins import get_plugin_class, MODULE_CACHE, PATH_CACHE, PLUGIN_P
 from ansible.utils.collection_loader import AnsibleCollectionConfig, AnsibleCollectionRef
 from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder, _get_collection_metadata
 from ansible.utils.display import Display
-from ansible.utils.plugin_docs import add_fragments, find_plugin_docfile
+from ansible.utils.plugin_docs import add_fragments
+from ansible.utils.unsafe_proxy import _is_unsafe
 
 # TODO: take the packaging dep, or vendor SpecifierSet?
 
@@ -46,6 +48,7 @@ except ImportError:
 
 import importlib.util
 
+_PLUGIN_FILTERS = defaultdict(frozenset)  # type: t.DefaultDict[str, frozenset]
 display = Display()
 
 get_with_context_result = namedtuple('get_with_context_result', ['object', 'plugin_load_context'])
@@ -236,6 +239,7 @@ class PluginLoader:
         self._module_cache = MODULE_CACHE[class_name]
         self._paths = PATH_CACHE[class_name]
         self._plugin_path_cache = PLUGIN_PATH_CACHE[class_name]
+        self._plugin_instance_cache = {} if self.subdir == 'vars_plugins' else None
 
         self._searched_paths = set()
 
@@ -260,6 +264,7 @@ class PluginLoader:
             self._module_cache = MODULE_CACHE[self.class_name]
             self._paths = PATH_CACHE[self.class_name]
             self._plugin_path_cache = PLUGIN_PATH_CACHE[self.class_name]
+            self._plugin_instance_cache = {} if self.subdir == 'vars_plugins' else None
             self._searched_paths = set()
 
     def __setstate__(self, data):
@@ -858,29 +863,52 @@ class PluginLoader:
 
     def get_with_context(self, name, *args, **kwargs):
         ''' instantiates a plugin of the given name using arguments '''
+        if _is_unsafe(name):
+            # Objects constructed using the name wrapped as unsafe remain
+            # (correctly) unsafe. Using such unsafe objects in places
+            # where underlying types (builtin string in this case) are
+            # expected can cause problems.
+            # One such case is importlib.abc.Loader.exec_module failing
+            # with "ValueError: unmarshallable object" because the module
+            # object is created with the __path__ attribute being wrapped
+            # as unsafe which isn't marshallable.
+            # Manually removing the unsafe wrapper prevents such issues.
+            name = name._strip_unsafe()
 
         found_in_cache = True
         class_only = kwargs.pop('class_only', False)
         collection_list = kwargs.pop('collection_list', None)
         if name in self.aliases:
             name = self.aliases[name]
+
+        if (cached_result := (self._plugin_instance_cache or {}).get(name)) and cached_result[1].resolved:
+            # Resolving the FQCN is slow, even if we've passed in the resolved FQCN.
+            # Short-circuit here if we've previously resolved this name.
+            # This will need to be restricted if non-vars plugins start using the cache, since
+            # some non-fqcn plugin need to be resolved again with the collections list.
+            return get_with_context_result(*cached_result)
+
         plugin_load_context = self.find_plugin_with_context(name, collection_list=collection_list)
         if not plugin_load_context.resolved or not plugin_load_context.plugin_resolved_path:
             # FIXME: this is probably an error (eg removed plugin)
             return get_with_context_result(None, plugin_load_context)
 
         fq_name = plugin_load_context.resolved_fqcn
-        if '.' not in fq_name:
+        if '.' not in fq_name and plugin_load_context.plugin_resolved_collection:
             fq_name = '.'.join((plugin_load_context.plugin_resolved_collection, fq_name))
-        name = plugin_load_context.plugin_resolved_name
+        resolved_type_name = plugin_load_context.plugin_resolved_name
         path = plugin_load_context.plugin_resolved_path
+        if (cached_result := (self._plugin_instance_cache or {}).get(fq_name)) and cached_result[1].resolved:
+            # This is unused by vars plugins, but it's here in case the instance cache expands to other plugin types.
+            # We get here if we've seen this plugin before, but it wasn't called with the resolved FQCN.
+            return get_with_context_result(*cached_result)
         redirected_names = plugin_load_context.redirect_list or []
 
         if path not in self._module_cache:
-            self._module_cache[path] = self._load_module_source(name, path)
+            self._module_cache[path] = self._load_module_source(resolved_type_name, path)
             found_in_cache = False
 
-        self._load_config_defs(name, self._module_cache[path], path)
+        self._load_config_defs(resolved_type_name, self._module_cache[path], path)
 
         obj = getattr(self._module_cache[path], self.class_name)
 
@@ -897,24 +925,29 @@ class PluginLoader:
                 return get_with_context_result(None, plugin_load_context)
 
         # FIXME: update this to use the load context
-        self._display_plugin_load(self.class_name, name, self._searched_paths, path, found_in_cache=found_in_cache, class_only=class_only)
+        self._display_plugin_load(self.class_name, resolved_type_name, self._searched_paths, path, found_in_cache=found_in_cache, class_only=class_only)
 
         if not class_only:
             try:
                 # A plugin may need to use its _load_name in __init__ (for example, to set
                 # or get options from config), so update the object before using the constructor
                 instance = object.__new__(obj)
-                self._update_object(instance, name, path, redirected_names, fq_name)
+                self._update_object(instance, resolved_type_name, path, redirected_names, fq_name)
                 obj.__init__(instance, *args, **kwargs)  # pylint: disable=unnecessary-dunder-call
                 obj = instance
             except TypeError as e:
                 if "abstract" in e.args[0]:
                     # Abstract Base Class or incomplete plugin, don't load
-                    display.v('Returning not found on "%s" as it has unimplemented abstract methods; %s' % (name, to_native(e)))
+                    display.v('Returning not found on "%s" as it has unimplemented abstract methods; %s' % (resolved_type_name, to_native(e)))
                     return get_with_context_result(None, plugin_load_context)
                 raise
 
-        self._update_object(obj, name, path, redirected_names, fq_name)
+        self._update_object(obj, resolved_type_name, path, redirected_names, fq_name)
+        if self._plugin_instance_cache is not None and getattr(obj, 'is_stateless', False):
+            self._plugin_instance_cache[fq_name] = (obj, plugin_load_context)
+        elif self._plugin_instance_cache is not None:
+            # The cache doubles as the load order, so record the FQCN even if the plugin hasn't set is_stateless = True
+            self._plugin_instance_cache[fq_name] = (None, PluginLoadContext())
         return get_with_context_result(obj, plugin_load_context)
 
     def _display_plugin_load(self, class_name, name, searched_paths, path, found_in_cache=None, class_only=None):
@@ -984,28 +1017,47 @@ class PluginLoader:
 
         loaded_modules = set()
         for path in all_matches:
+
             name = os.path.splitext(path)[0]
             basename = os.path.basename(name)
+            is_j2 = isinstance(self, Jinja2Loader)
 
-            if basename in _PLUGIN_FILTERS[self.package]:
+            if is_j2:
+                ref_name = path
+            else:
+                ref_name = basename
+
+            if not is_j2 and basename in _PLUGIN_FILTERS[self.package]:
+                # j2 plugins get processed in own class, here they would just be container files
                 display.debug("'%s' skipped due to a defined plugin filter" % basename)
                 continue
 
             if basename == '__init__' or (basename == 'base' and self.package == 'ansible.plugins.cache'):
                 # cache has legacy 'base.py' file, which is wrapper for __init__.py
-                display.debug("'%s' skipped due to reserved name" % basename)
+                display.debug("'%s' skipped due to reserved name" % name)
                 continue
 
-            if dedupe and basename in loaded_modules:
-                display.debug("'%s' skipped as duplicate" % basename)
+            if dedupe and ref_name in loaded_modules:
+                # for j2 this is 'same file', other plugins it is basename
+                display.debug("'%s' skipped as duplicate" % ref_name)
                 continue
 
-            loaded_modules.add(basename)
+            loaded_modules.add(ref_name)
 
             if path_only:
                 yield path
                 continue
 
+            if path in legacy_excluding_builtin:
+                fqcn = basename
+            else:
+                fqcn = f"ansible.builtin.{basename}"
+
+            if (cached_result := (self._plugin_instance_cache or {}).get(fqcn)) and cached_result[1].resolved:
+                # Here just in case, but we don't call all() multiple times for vars plugins, so this should not be used.
+                yield cached_result[0]
+                continue
+
             if path not in self._module_cache:
                 if self.type in ('filter', 'test'):
                     # filter and test plugin files can contain multiple plugins
@@ -1053,11 +1105,20 @@ class PluginLoader:
                 except TypeError as e:
                     display.warning("Skipping plugin (%s) as it seems to be incomplete: %s" % (path, to_text(e)))
 
-            if path in legacy_excluding_builtin:
-                fqcn = basename
-            else:
-                fqcn = f"ansible.builtin.{basename}"
             self._update_object(obj, basename, path, resolved=fqcn)
+
+            if self._plugin_instance_cache is not None:
+                needs_enabled = False
+                if hasattr(obj, 'REQUIRES_ENABLED'):
+                    needs_enabled = obj.REQUIRES_ENABLED
+                elif hasattr(obj, 'REQUIRES_WHITELIST'):
+                    needs_enabled = obj.REQUIRES_WHITELIST
+                    display.deprecated("The VarsModule class variable 'REQUIRES_WHITELIST' is deprecated. "
+                                       "Use 'REQUIRES_ENABLED' instead.", version=2.18)
+                if not needs_enabled:
+                    # Use get_with_context to cache the plugin the first time we see it.
+                    self.get_with_context(fqcn)[0]
+
             yield obj
 
 
@@ -1333,7 +1394,7 @@ def get_fqcr_and_name(resource, collection='ansible.builtin'):
 
 
 def _load_plugin_filter():
-    filters = defaultdict(frozenset)
+    filters = _PLUGIN_FILTERS
     user_set = False
     if C.PLUGIN_FILTERS_CFG is None:
         filter_cfg = '/etc/ansible/plugin_filters.yml'
@@ -1361,15 +1422,21 @@ def _load_plugin_filter():
         version = to_text(version)
         version = version.strip()
 
+        # Modules and action plugins share the same reject list since the difference between the
+        # two isn't visible to the users
         if version == u'1.0':
-            # Modules and action plugins share the same blacklist since the difference between the
-            # two isn't visible to the users
+
+            if 'module_blacklist' in filter_data:
+                display.deprecated("'module_blacklist' is being removed in favor of 'module_rejectlist'", version='2.18')
+                if 'module_rejectlist' not in filter_data:
+                    filter_data['module_rejectlist'] = filter_data['module_blacklist']
+                del filter_data['module_blacklist']
+
             try:
-                # reject list was documented but we never changed the code from blacklist, will be deprected in 2.15
-                filters['ansible.modules'] = frozenset(filter_data.get('module_rejectlist)', filter_data['module_blacklist']))
+                filters['ansible.modules'] = frozenset(filter_data['module_rejectlist'])
             except TypeError:
                 display.warning(u'Unable to parse the plugin filter file {0} as'
-                                u' module_blacklist is not a list.'
+                                u' module_rejectlist is not a list.'
                                 u' Skipping.'.format(filter_cfg))
                 return filters
             filters['ansible.plugins.action'] = filters['ansible.modules']
@@ -1381,11 +1448,11 @@ def _load_plugin_filter():
             display.warning(u'The plugin filter file, {0} does not exist.'
                             u' Skipping.'.format(filter_cfg))
 
-    # Specialcase the stat module as Ansible can run very few things if stat is blacklisted.
+    # Specialcase the stat module as Ansible can run very few things if stat is rejected
     if 'stat' in filters['ansible.modules']:
-        raise AnsibleError('The stat module was specified in the module blacklist file, {0}, but'
+        raise AnsibleError('The stat module was specified in the module reject list file, {0}, but'
                            ' Ansible will not function without the stat module.  Please remove stat'
-                           ' from the blacklist.'.format(to_native(filter_cfg)))
+                           ' from the reject list.'.format(to_native(filter_cfg)))
     return filters
 
 
@@ -1425,25 +1492,38 @@ def _does_collection_support_ansible_version(requirement_string, ansible_version
     return ss.contains(base_ansible_version)
 
 
-def _configure_collection_loader():
+def _configure_collection_loader(prefix_collections_path=None):
     if AnsibleCollectionConfig.collection_finder:
         # this must be a Python warning so that it can be filtered out by the import sanity test
         warnings.warn('AnsibleCollectionFinder has already been configured')
         return
 
-    finder = _AnsibleCollectionFinder(C.COLLECTIONS_PATHS, C.COLLECTIONS_SCAN_SYS_PATH)
+    if prefix_collections_path is None:
+        prefix_collections_path = []
+
+    paths = list(prefix_collections_path) + C.COLLECTIONS_PATHS
+    finder = _AnsibleCollectionFinder(paths, C.COLLECTIONS_SCAN_SYS_PATH)
     finder._install()
 
     # this should succeed now
     AnsibleCollectionConfig.on_collection_load += _on_collection_load_handler
 
 
-# TODO: All of the following is initialization code   It should be moved inside of an initialization
-# function which is called at some point early in the ansible and ansible-playbook CLI startup.
+def init_plugin_loader(prefix_collections_path=None):
+    """Initialize the plugin filters and the collection loaders
+
+    This method must be called to configure and insert the collection python loaders
+    into ``sys.meta_path`` and ``sys.path_hooks``.
+
+    This method is only called in ``CLI.run`` after CLI args have been parsed, so that
+    instantiation of the collection finder can utilize parsed CLI args, and to not cause
+    side effects.
+    """
+    _load_plugin_filter()
+    _configure_collection_loader(prefix_collections_path)
 
-_PLUGIN_FILTERS = _load_plugin_filter()
 
-_configure_collection_loader()
+# TODO: Evaluate making these class instantiations lazy, but keep them in the global scope
 
 # doc fragments first
 fragment_loader = PluginLoader(