diff options
Diffstat (limited to 'lib/ansible/plugins/loader.py')
| -rw-r--r-- | lib/ansible/plugins/loader.py | 148 |
1 files changed, 114 insertions, 34 deletions
diff --git a/lib/ansible/plugins/loader.py b/lib/ansible/plugins/loader.py index 8b7fbfce..9ff19bbb 100644 --- a/lib/ansible/plugins/loader.py +++ b/lib/ansible/plugins/loader.py @@ -17,6 +17,7 @@ import warnings from collections import defaultdict, namedtuple from traceback import format_exc +import ansible.module_utils.compat.typing as t from .filter import AnsibleJinja2Filter from .test import AnsibleJinja2Test @@ -24,7 +25,7 @@ from .test import AnsibleJinja2Test from ansible import __version__ as ansible_version from ansible import constants as C from ansible.errors import AnsibleError, AnsiblePluginCircularRedirect, AnsiblePluginRemovedError, AnsibleCollectionUnsupportedVersionError -from ansible.module_utils._text import to_bytes, to_text, to_native +from ansible.module_utils.common.text.converters import to_bytes, to_text, to_native from ansible.module_utils.compat.importlib import import_module from ansible.module_utils.six import string_types from ansible.parsing.utils.yaml import from_yaml @@ -33,7 +34,8 @@ from ansible.plugins import get_plugin_class, MODULE_CACHE, PATH_CACHE, PLUGIN_P from ansible.utils.collection_loader import AnsibleCollectionConfig, AnsibleCollectionRef from ansible.utils.collection_loader._collection_finder import _AnsibleCollectionFinder, _get_collection_metadata from ansible.utils.display import Display -from ansible.utils.plugin_docs import add_fragments, find_plugin_docfile +from ansible.utils.plugin_docs import add_fragments +from ansible.utils.unsafe_proxy import _is_unsafe # TODO: take the packaging dep, or vendor SpecifierSet? @@ -46,6 +48,7 @@ except ImportError: import importlib.util +_PLUGIN_FILTERS = defaultdict(frozenset) # type: t.DefaultDict[str, frozenset] display = Display() get_with_context_result = namedtuple('get_with_context_result', ['object', 'plugin_load_context']) @@ -236,6 +239,7 @@ class PluginLoader: self._module_cache = MODULE_CACHE[class_name] self._paths = PATH_CACHE[class_name] self._plugin_path_cache = PLUGIN_PATH_CACHE[class_name] + self._plugin_instance_cache = {} if self.subdir == 'vars_plugins' else None self._searched_paths = set() @@ -260,6 +264,7 @@ class PluginLoader: self._module_cache = MODULE_CACHE[self.class_name] self._paths = PATH_CACHE[self.class_name] self._plugin_path_cache = PLUGIN_PATH_CACHE[self.class_name] + self._plugin_instance_cache = {} if self.subdir == 'vars_plugins' else None self._searched_paths = set() def __setstate__(self, data): @@ -858,29 +863,52 @@ class PluginLoader: def get_with_context(self, name, *args, **kwargs): ''' instantiates a plugin of the given name using arguments ''' + if _is_unsafe(name): + # Objects constructed using the name wrapped as unsafe remain + # (correctly) unsafe. Using such unsafe objects in places + # where underlying types (builtin string in this case) are + # expected can cause problems. + # One such case is importlib.abc.Loader.exec_module failing + # with "ValueError: unmarshallable object" because the module + # object is created with the __path__ attribute being wrapped + # as unsafe which isn't marshallable. + # Manually removing the unsafe wrapper prevents such issues. + name = name._strip_unsafe() found_in_cache = True class_only = kwargs.pop('class_only', False) collection_list = kwargs.pop('collection_list', None) if name in self.aliases: name = self.aliases[name] + + if (cached_result := (self._plugin_instance_cache or {}).get(name)) and cached_result[1].resolved: + # Resolving the FQCN is slow, even if we've passed in the resolved FQCN. + # Short-circuit here if we've previously resolved this name. + # This will need to be restricted if non-vars plugins start using the cache, since + # some non-fqcn plugin need to be resolved again with the collections list. + return get_with_context_result(*cached_result) + plugin_load_context = self.find_plugin_with_context(name, collection_list=collection_list) if not plugin_load_context.resolved or not plugin_load_context.plugin_resolved_path: # FIXME: this is probably an error (eg removed plugin) return get_with_context_result(None, plugin_load_context) fq_name = plugin_load_context.resolved_fqcn - if '.' not in fq_name: + if '.' not in fq_name and plugin_load_context.plugin_resolved_collection: fq_name = '.'.join((plugin_load_context.plugin_resolved_collection, fq_name)) - name = plugin_load_context.plugin_resolved_name + resolved_type_name = plugin_load_context.plugin_resolved_name path = plugin_load_context.plugin_resolved_path + if (cached_result := (self._plugin_instance_cache or {}).get(fq_name)) and cached_result[1].resolved: + # This is unused by vars plugins, but it's here in case the instance cache expands to other plugin types. + # We get here if we've seen this plugin before, but it wasn't called with the resolved FQCN. + return get_with_context_result(*cached_result) redirected_names = plugin_load_context.redirect_list or [] if path not in self._module_cache: - self._module_cache[path] = self._load_module_source(name, path) + self._module_cache[path] = self._load_module_source(resolved_type_name, path) found_in_cache = False - self._load_config_defs(name, self._module_cache[path], path) + self._load_config_defs(resolved_type_name, self._module_cache[path], path) obj = getattr(self._module_cache[path], self.class_name) @@ -897,24 +925,29 @@ class PluginLoader: return get_with_context_result(None, plugin_load_context) # FIXME: update this to use the load context - self._display_plugin_load(self.class_name, name, self._searched_paths, path, found_in_cache=found_in_cache, class_only=class_only) + self._display_plugin_load(self.class_name, resolved_type_name, self._searched_paths, path, found_in_cache=found_in_cache, class_only=class_only) if not class_only: try: # A plugin may need to use its _load_name in __init__ (for example, to set # or get options from config), so update the object before using the constructor instance = object.__new__(obj) - self._update_object(instance, name, path, redirected_names, fq_name) + self._update_object(instance, resolved_type_name, path, redirected_names, fq_name) obj.__init__(instance, *args, **kwargs) # pylint: disable=unnecessary-dunder-call obj = instance except TypeError as e: if "abstract" in e.args[0]: # Abstract Base Class or incomplete plugin, don't load - display.v('Returning not found on "%s" as it has unimplemented abstract methods; %s' % (name, to_native(e))) + display.v('Returning not found on "%s" as it has unimplemented abstract methods; %s' % (resolved_type_name, to_native(e))) return get_with_context_result(None, plugin_load_context) raise - self._update_object(obj, name, path, redirected_names, fq_name) + self._update_object(obj, resolved_type_name, path, redirected_names, fq_name) + if self._plugin_instance_cache is not None and getattr(obj, 'is_stateless', False): + self._plugin_instance_cache[fq_name] = (obj, plugin_load_context) + elif self._plugin_instance_cache is not None: + # The cache doubles as the load order, so record the FQCN even if the plugin hasn't set is_stateless = True + self._plugin_instance_cache[fq_name] = (None, PluginLoadContext()) return get_with_context_result(obj, plugin_load_context) def _display_plugin_load(self, class_name, name, searched_paths, path, found_in_cache=None, class_only=None): @@ -984,28 +1017,47 @@ class PluginLoader: loaded_modules = set() for path in all_matches: + name = os.path.splitext(path)[0] basename = os.path.basename(name) + is_j2 = isinstance(self, Jinja2Loader) - if basename in _PLUGIN_FILTERS[self.package]: + if is_j2: + ref_name = path + else: + ref_name = basename + + if not is_j2 and basename in _PLUGIN_FILTERS[self.package]: + # j2 plugins get processed in own class, here they would just be container files display.debug("'%s' skipped due to a defined plugin filter" % basename) continue if basename == '__init__' or (basename == 'base' and self.package == 'ansible.plugins.cache'): # cache has legacy 'base.py' file, which is wrapper for __init__.py - display.debug("'%s' skipped due to reserved name" % basename) + display.debug("'%s' skipped due to reserved name" % name) continue - if dedupe and basename in loaded_modules: - display.debug("'%s' skipped as duplicate" % basename) + if dedupe and ref_name in loaded_modules: + # for j2 this is 'same file', other plugins it is basename + display.debug("'%s' skipped as duplicate" % ref_name) continue - loaded_modules.add(basename) + loaded_modules.add(ref_name) if path_only: yield path continue + if path in legacy_excluding_builtin: + fqcn = basename + else: + fqcn = f"ansible.builtin.{basename}" + + if (cached_result := (self._plugin_instance_cache or {}).get(fqcn)) and cached_result[1].resolved: + # Here just in case, but we don't call all() multiple times for vars plugins, so this should not be used. + yield cached_result[0] + continue + if path not in self._module_cache: if self.type in ('filter', 'test'): # filter and test plugin files can contain multiple plugins @@ -1053,11 +1105,20 @@ class PluginLoader: except TypeError as e: display.warning("Skipping plugin (%s) as it seems to be incomplete: %s" % (path, to_text(e))) - if path in legacy_excluding_builtin: - fqcn = basename - else: - fqcn = f"ansible.builtin.{basename}" self._update_object(obj, basename, path, resolved=fqcn) + + if self._plugin_instance_cache is not None: + needs_enabled = False + if hasattr(obj, 'REQUIRES_ENABLED'): + needs_enabled = obj.REQUIRES_ENABLED + elif hasattr(obj, 'REQUIRES_WHITELIST'): + needs_enabled = obj.REQUIRES_WHITELIST + display.deprecated("The VarsModule class variable 'REQUIRES_WHITELIST' is deprecated. " + "Use 'REQUIRES_ENABLED' instead.", version=2.18) + if not needs_enabled: + # Use get_with_context to cache the plugin the first time we see it. + self.get_with_context(fqcn)[0] + yield obj @@ -1333,7 +1394,7 @@ def get_fqcr_and_name(resource, collection='ansible.builtin'): def _load_plugin_filter(): - filters = defaultdict(frozenset) + filters = _PLUGIN_FILTERS user_set = False if C.PLUGIN_FILTERS_CFG is None: filter_cfg = '/etc/ansible/plugin_filters.yml' @@ -1361,15 +1422,21 @@ def _load_plugin_filter(): version = to_text(version) version = version.strip() + # Modules and action plugins share the same reject list since the difference between the + # two isn't visible to the users if version == u'1.0': - # Modules and action plugins share the same blacklist since the difference between the - # two isn't visible to the users + + if 'module_blacklist' in filter_data: + display.deprecated("'module_blacklist' is being removed in favor of 'module_rejectlist'", version='2.18') + if 'module_rejectlist' not in filter_data: + filter_data['module_rejectlist'] = filter_data['module_blacklist'] + del filter_data['module_blacklist'] + try: - # reject list was documented but we never changed the code from blacklist, will be deprected in 2.15 - filters['ansible.modules'] = frozenset(filter_data.get('module_rejectlist)', filter_data['module_blacklist'])) + filters['ansible.modules'] = frozenset(filter_data['module_rejectlist']) except TypeError: display.warning(u'Unable to parse the plugin filter file {0} as' - u' module_blacklist is not a list.' + u' module_rejectlist is not a list.' u' Skipping.'.format(filter_cfg)) return filters filters['ansible.plugins.action'] = filters['ansible.modules'] @@ -1381,11 +1448,11 @@ def _load_plugin_filter(): display.warning(u'The plugin filter file, {0} does not exist.' u' Skipping.'.format(filter_cfg)) - # Specialcase the stat module as Ansible can run very few things if stat is blacklisted. + # Specialcase the stat module as Ansible can run very few things if stat is rejected if 'stat' in filters['ansible.modules']: - raise AnsibleError('The stat module was specified in the module blacklist file, {0}, but' + raise AnsibleError('The stat module was specified in the module reject list file, {0}, but' ' Ansible will not function without the stat module. Please remove stat' - ' from the blacklist.'.format(to_native(filter_cfg))) + ' from the reject list.'.format(to_native(filter_cfg))) return filters @@ -1425,25 +1492,38 @@ def _does_collection_support_ansible_version(requirement_string, ansible_version return ss.contains(base_ansible_version) -def _configure_collection_loader(): +def _configure_collection_loader(prefix_collections_path=None): if AnsibleCollectionConfig.collection_finder: # this must be a Python warning so that it can be filtered out by the import sanity test warnings.warn('AnsibleCollectionFinder has already been configured') return - finder = _AnsibleCollectionFinder(C.COLLECTIONS_PATHS, C.COLLECTIONS_SCAN_SYS_PATH) + if prefix_collections_path is None: + prefix_collections_path = [] + + paths = list(prefix_collections_path) + C.COLLECTIONS_PATHS + finder = _AnsibleCollectionFinder(paths, C.COLLECTIONS_SCAN_SYS_PATH) finder._install() # this should succeed now AnsibleCollectionConfig.on_collection_load += _on_collection_load_handler -# TODO: All of the following is initialization code It should be moved inside of an initialization -# function which is called at some point early in the ansible and ansible-playbook CLI startup. +def init_plugin_loader(prefix_collections_path=None): + """Initialize the plugin filters and the collection loaders + + This method must be called to configure and insert the collection python loaders + into ``sys.meta_path`` and ``sys.path_hooks``. + + This method is only called in ``CLI.run`` after CLI args have been parsed, so that + instantiation of the collection finder can utilize parsed CLI args, and to not cause + side effects. + """ + _load_plugin_filter() + _configure_collection_loader(prefix_collections_path) -_PLUGIN_FILTERS = _load_plugin_filter() -_configure_collection_loader() +# TODO: Evaluate making these class instantiations lazy, but keep them in the global scope # doc fragments first fragment_loader = PluginLoader( |