New upstream version 2.13.0

18 files changed, 217 insertions, 169 deletions

diff --git a/lib/ansible/utils/_junit_xml.py b/lib/ansible/utils/_junit_xml.py
index a21fed4c..3b958672 100644
--- a/lib/ansible/utils/_junit_xml.py
+++ b/lib/ansible/utils/_junit_xml.py
@@ -8,19 +8,18 @@ import abc
 import dataclasses
 import datetime
 import decimal
-import typing as t
 
 from xml.dom import minidom
 # noinspection PyPep8Naming
 from xml.etree import ElementTree as ET
 
 
-@dataclasses.dataclass
+@dataclasses.dataclass  # type: ignore[misc]  # https://github.com/python/mypy/issues/5374
 class TestResult(metaclass=abc.ABCMeta):
     """Base class for the result of a test case."""
-    output: t.Optional[str] = None
-    message: t.Optional[str] = None
-    type: t.Optional[str] = None
+    output: str | None = None
+    message: str | None = None
+    type: str | None = None
 
     def __post_init__(self):
         if self.type is None:
@@ -31,7 +30,7 @@ class TestResult(metaclass=abc.ABCMeta):
     def tag(self) -> str:
         """Tag name for the XML element created by this result type."""
 
-    def get_attributes(self) -> t.Dict[str, str]:
+    def get_attributes(self) -> dict[str, str]:
         """Return a dictionary of attributes for this instance."""
         return _attributes(
             message=self.message,
@@ -68,16 +67,16 @@ class TestError(TestResult):
 class TestCase:
     """An individual test case."""
     name: str
-    assertions: t.Optional[int] = None
-    classname: t.Optional[str] = None
-    status: t.Optional[str] = None
-    time: t.Optional[decimal.Decimal] = None
+    assertions: int | None = None
+    classname: str | None = None
+    status: str | None = None
+    time: decimal.Decimal | None = None
 
-    errors: t.List[TestError] = dataclasses.field(default_factory=list)
-    failures: t.List[TestFailure] = dataclasses.field(default_factory=list)
-    skipped: t.Optional[str] = None
-    system_out: t.Optional[str] = None
-    system_err: t.Optional[str] = None
+    errors: list[TestError] = dataclasses.field(default_factory=list)
+    failures: list[TestFailure] = dataclasses.field(default_factory=list)
+    skipped: str | None = None
+    system_out: str | None = None
+    system_err: str | None = None
 
     is_disabled: bool = False
 
@@ -96,7 +95,7 @@ class TestCase:
         """True if the test case was skipped."""
         return bool(self.skipped)
 
-    def get_attributes(self) -> t.Dict[str, str]:
+    def get_attributes(self) -> dict[str, str]:
         """Return a dictionary of attributes for this instance."""
         return _attributes(
             assertions=self.assertions,
@@ -129,15 +128,15 @@ class TestCase:
 class TestSuite:
     """A collection of test cases."""
     name: str
-    hostname: t.Optional[str] = None
-    id: t.Optional[str] = None
-    package: t.Optional[str] = None
-    timestamp: t.Optional[datetime.datetime] = None
+    hostname: str | None = None
+    id: str | None = None
+    package: str | None = None
+    timestamp: datetime.datetime | None = None
 
-    properties: t.Dict[str, str] = dataclasses.field(default_factory=dict)
-    cases: t.List[TestCase] = dataclasses.field(default_factory=list)
-    system_out: t.Optional[str] = None
-    system_err: t.Optional[str] = None
+    properties: dict[str, str] = dataclasses.field(default_factory=dict)
+    cases: list[TestCase] = dataclasses.field(default_factory=list)
+    system_out: str | None = None
+    system_err: str | None = None
 
     @property
     def disabled(self) -> int:
@@ -167,9 +166,9 @@ class TestSuite:
     @property
     def time(self) -> decimal.Decimal:
         """The total time from all test cases."""
-        return sum(case.time for case in self.cases if case.time)
+        return decimal.Decimal(sum(case.time for case in self.cases if case.time))
 
-    def get_attributes(self) -> t.Dict[str, str]:
+    def get_attributes(self) -> dict[str, str]:
         """Return a dictionary of attributes for this instance."""
         return _attributes(
             disabled=self.disabled,
@@ -206,9 +205,9 @@ class TestSuite:
 @dataclasses.dataclass
 class TestSuites:
     """A collection of test suites."""
-    name: t.Optional[str] = None
+    name: str | None = None
 
-    suites: t.List[TestSuite] = dataclasses.field(default_factory=list)
+    suites: list[TestSuite] = dataclasses.field(default_factory=list)
 
     @property
     def disabled(self) -> int:
@@ -233,9 +232,9 @@ class TestSuites:
     @property
     def time(self) -> decimal.Decimal:
         """The total time from all test cases."""
-        return sum(suite.time for suite in self.suites)
+        return decimal.Decimal(sum(suite.time for suite in self.suites))
 
-    def get_attributes(self) -> t.Dict[str, str]:
+    def get_attributes(self) -> dict[str, str]:
         """Return a dictionary of attributes for this instance."""
         return _attributes(
             disabled=self.disabled,
@@ -258,7 +257,7 @@ class TestSuites:
         return _pretty_xml(self.get_xml_element())
 
 
-def _attributes(**kwargs) -> t.Dict[str, str]:
+def _attributes(**kwargs) -> dict[str, str]:
     """Return the given kwargs as a dictionary with values converted to strings. Items with a value of None will be omitted."""
     return {key: str(value) for key, value in kwargs.items() if value is not None}
 
diff --git a/lib/ansible/utils/cmd_functions.py b/lib/ansible/utils/cmd_functions.py
index 7a0fb23e..d4edb2f4 100644
--- a/lib/ansible/utils/cmd_functions.py
+++ b/lib/ansible/utils/cmd_functions.py
@@ -24,21 +24,13 @@ import shlex
 import subprocess
 import sys
 
-from ansible.module_utils.six import PY2, PY3
 from ansible.module_utils._text import to_bytes
 
 
 def run_cmd(cmd, live=False, readsize=10):
-
-    # readsize = 10
-
-    # On python2, shlex needs byte strings
-    if PY2:
-        cmd = to_bytes(cmd, errors='surrogate_or_strict')
     cmdargs = shlex.split(cmd)
 
-    # subprocess should be passed byte strings.  (on python2.6 it must be
-    # passed byte strtings)
+    # subprocess should be passed byte strings.
     cmdargs = [to_bytes(a, errors='surrogate_or_strict') for a in cmdargs]
 
     p = subprocess.Popen(cmdargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -52,11 +44,7 @@ def run_cmd(cmd, live=False, readsize=10):
         if p.stdout in rfd:
             dat = os.read(p.stdout.fileno(), readsize)
             if live:
-                # On python3, stdout has a codec to go from text type to bytes
-                if PY3:
-                    sys.stdout.buffer.write(dat)
-                else:
-                    sys.stdout.write(dat)
+                sys.stdout.buffer.write(dat)
             stdout += dat
             if dat == b'':
                 rpipes.remove(p.stdout)
@@ -64,11 +52,7 @@ def run_cmd(cmd, live=False, readsize=10):
             dat = os.read(p.stderr.fileno(), readsize)
             stderr += dat
             if live:
-                # On python3, stdout has a codec to go from text type to bytes
-                if PY3:
-                    sys.stdout.buffer.write(dat)
-                else:
-                    sys.stdout.write(dat)
+                sys.stdout.buffer.write(dat)
             if dat == b'':
                 rpipes.remove(p.stderr)
         # only break out if we've emptied the pipes, or there is nothing to
diff --git a/lib/ansible/utils/collection_loader/__init__.py b/lib/ansible/utils/collection_loader/__init__.py
index 21c49c47..83cc2462 100644
--- a/lib/ansible/utils/collection_loader/__init__.py
+++ b/lib/ansible/utils/collection_loader/__init__.py
@@ -1,13 +1,8 @@
 # (c) 2019 Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
-# CAUTION: There are two implementations of the collection loader.
-#          They must be kept functionally identical, although their implementations may differ.
-#
-# 1) The controller implementation resides in the "lib/ansible/utils/collection_loader/" directory.
-#    It must function on all Python versions supported on the controller.
-# 2) The ansible-test implementation resides in the "test/lib/ansible_test/_util/target/legacy_collection_loader/" directory.
-#    It must function on all Python versions supported on managed hosts which are not supported by the controller.
+# CAUTION: This implementation of the collection loader is used by ansible-test.
+#          Because of this, it must be compatible with all Python versions supported on the controller or remote.
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
diff --git a/lib/ansible/utils/collection_loader/_collection_config.py b/lib/ansible/utils/collection_loader/_collection_config.py
index a2031931..4f73a1a7 100644
--- a/lib/ansible/utils/collection_loader/_collection_config.py
+++ b/lib/ansible/utils/collection_loader/_collection_config.py
@@ -1,19 +1,14 @@
 # (c) 2019 Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
-# CAUTION: There are two implementations of the collection loader.
-#          They must be kept functionally identical, although their implementations may differ.
-#
-# 1) The controller implementation resides in the "lib/ansible/utils/collection_loader/" directory.
-#    It must function on all Python versions supported on the controller.
-# 2) The ansible-test implementation resides in the "test/lib/ansible_test/_util/target/legacy_collection_loader/" directory.
-#    It must function on all Python versions supported on managed hosts which are not supported by the controller.
+# CAUTION: This implementation of the collection loader is used by ansible-test.
+#          Because of this, it must be compatible with all Python versions supported on the controller or remote.
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 from ansible.module_utils.common.text.converters import to_text
-from ansible.module_utils.six import with_metaclass
+from ansible.module_utils.six import add_metaclass
 
 
 class _EventSource:
@@ -103,5 +98,6 @@ class _AnsibleCollectionConfig(type):
 
 
 # concrete class of our metaclass type that defines the class properties we want
-class AnsibleCollectionConfig(with_metaclass(_AnsibleCollectionConfig)):
+@add_metaclass(_AnsibleCollectionConfig)
+class AnsibleCollectionConfig(object):
     pass
diff --git a/lib/ansible/utils/collection_loader/_collection_finder.py b/lib/ansible/utils/collection_loader/_collection_finder.py
index 13fab892..c581abf2 100644
--- a/lib/ansible/utils/collection_loader/_collection_finder.py
+++ b/lib/ansible/utils/collection_loader/_collection_finder.py
@@ -1,13 +1,8 @@
 # (c) 2019 Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
-# CAUTION: There are two implementations of the collection loader.
-#          They must be kept functionally identical, although their implementations may differ.
-#
-# 1) The controller implementation resides in the "lib/ansible/utils/collection_loader/" directory.
-#    It must function on all Python versions supported on the controller.
-# 2) The ansible-test implementation resides in the "test/lib/ansible_test/_util/target/legacy_collection_loader/" directory.
-#    It must function on all Python versions supported on managed hosts which are not supported by the controller.
+# CAUTION: This implementation of the collection loader is used by ansible-test.
+#          Because of this, it must be compatible with all Python versions supported on the controller or remote.
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
@@ -33,7 +28,7 @@ from types import ModuleType
 try:
     from importlib import import_module
 except ImportError:
-    def import_module(name):
+    def import_module(name):  # type: ignore[misc]
         __import__(name)
         return sys.modules[name]
 
@@ -41,7 +36,19 @@ try:
     from importlib import reload as reload_module
 except ImportError:
     # 2.7 has a global reload function instead...
-    reload_module = reload  # pylint:disable=undefined-variable
+    reload_module = reload  # type: ignore[name-defined]  # pylint:disable=undefined-variable
+
+try:
+    from importlib.util import spec_from_loader
+except ImportError:
+    pass
+
+try:
+    from importlib.machinery import FileFinder
+except ImportError:
+    HAS_FILE_FINDER = False
+else:
+    HAS_FILE_FINDER = True
 
 # NB: this supports import sanity test providing a different impl
 try:
@@ -64,10 +71,10 @@ try:  # NOTE: py3/py2 compat
     # py2 mypy can't deal with try/excepts
     is_python_identifier = str.isidentifier  # type: ignore[attr-defined]
 except AttributeError:  # Python 2
-    def is_python_identifier(tested_str):  # type: (str) -> bool
+    def is_python_identifier(self):  # type: (str) -> bool
         """Determine whether the given string is a Python identifier."""
         # Ref: https://stackoverflow.com/a/55802320/595220
-        return bool(re.match(_VALID_IDENTIFIER_STRING_REGEX, tested_str))
+        return bool(re.match(_VALID_IDENTIFIER_STRING_REGEX, self))
 
 
 PB_EXTENSIONS = ('.yml', '.yaml')
@@ -184,9 +191,7 @@ class _AnsibleCollectionFinder:
             return
         reload_module(m)
 
-    def find_module(self, fullname, path=None):
-        # Figure out what's being asked for, and delegate to a special-purpose loader
-
+    def _get_loader(self, fullname, path=None):
         split_name = fullname.split('.')
         toplevel_pkg = split_name[0]
         module_to_find = split_name[-1]
@@ -207,23 +212,41 @@ class _AnsibleCollectionFinder:
         if part_count > 1 and path is None:
             raise ValueError('path must be specified for subpackages (trying to find {0})'.format(fullname))
 
+        if toplevel_pkg == 'ansible':
+            # something under the ansible package, delegate to our internal loader in case of redirections
+            initialize_loader = _AnsibleInternalRedirectLoader
+        elif part_count == 1:
+            initialize_loader = _AnsibleCollectionRootPkgLoader
+        elif part_count == 2:  # ns pkg eg, ansible_collections, ansible_collections.somens
+            initialize_loader = _AnsibleCollectionNSPkgLoader
+        elif part_count == 3:  # collection pkg eg, ansible_collections.somens.somecoll
+            initialize_loader = _AnsibleCollectionPkgLoader
+        else:
+            # anything below the collection
+            initialize_loader = _AnsibleCollectionLoader
+
         # NB: actual "find"ing is delegated to the constructors on the various loaders; they'll ImportError if not found
         try:
-            if toplevel_pkg == 'ansible':
-                # something under the ansible package, delegate to our internal loader in case of redirections
-                return _AnsibleInternalRedirectLoader(fullname=fullname, path_list=path)
-            if part_count == 1:
-                return _AnsibleCollectionRootPkgLoader(fullname=fullname, path_list=path)
-            if part_count == 2:  # ns pkg eg, ansible_collections, ansible_collections.somens
-                return _AnsibleCollectionNSPkgLoader(fullname=fullname, path_list=path)
-            elif part_count == 3:  # collection pkg eg, ansible_collections.somens.somecoll
-                return _AnsibleCollectionPkgLoader(fullname=fullname, path_list=path)
-            # anything below the collection
-            return _AnsibleCollectionLoader(fullname=fullname, path_list=path)
+            return initialize_loader(fullname=fullname, path_list=path)
         except ImportError:
             # TODO: log attempt to load context
             return None
 
+    def find_module(self, fullname, path=None):
+        # Figure out what's being asked for, and delegate to a special-purpose loader
+        return self._get_loader(fullname, path)
+
+    def find_spec(self, fullname, path, target=None):
+        loader = self._get_loader(fullname, path)
+
+        if loader is None:
+            return None
+
+        spec = spec_from_loader(fullname, loader)
+        if spec is not None and hasattr(loader, '_subpackage_search_paths'):
+            spec.submodule_search_locations = loader._subpackage_search_paths
+        return spec
+
 
 # Implements a path_hook finder for iter_modules (since it's only path based). This finder does not need to actually
 # function as a finder in most cases, since our meta_path finder is consulted first for *almost* everything, except
@@ -251,14 +274,13 @@ class _AnsiblePathHookFinder:
 
     _filefinder_path_hook = _get_filefinder_path_hook()
 
-    def find_module(self, fullname, path=None):
-        # we ignore the passed in path here- use what we got from the path hook init
+    def _get_finder(self, fullname):
         split_name = fullname.split('.')
         toplevel_pkg = split_name[0]
 
         if toplevel_pkg == 'ansible_collections':
             # collections content? delegate to the collection finder
-            return self._collection_finder.find_module(fullname, path=[self._pathctx])
+            return self._collection_finder
         else:
             # Something else; we'd normally restrict this to `ansible` descendent modules so that any weird loader
             # behavior that arbitrary Python modules have can be serviced by those loaders. In some dev/test
@@ -277,13 +299,37 @@ class _AnsiblePathHookFinder:
                         # might not be in some other situation...
                         return None
 
-                spec = self._file_finder.find_spec(fullname)
-                if not spec:
-                    return None
-                return spec.loader
-            else:
-                # call py2's internal loader
-                return pkgutil.ImpImporter(self._pathctx).find_module(fullname)
+                return self._file_finder
+
+            # call py2's internal loader
+            return pkgutil.ImpImporter(self._pathctx)
+
+    def find_module(self, fullname, path=None):
+        # we ignore the passed in path here- use what we got from the path hook init
+        finder = self._get_finder(fullname)
+
+        if finder is None:
+            return None
+        elif HAS_FILE_FINDER and isinstance(finder, FileFinder):
+            # this codepath is erroneously used under some cases in py3,
+            # and the find_module method on FileFinder does not accept the path arg
+            # see https://github.com/pypa/setuptools/pull/2918
+            return finder.find_module(fullname)
+        else:
+            return finder.find_module(fullname, path=[self._pathctx])
+
+    def find_spec(self, fullname, target=None):
+        split_name = fullname.split('.')
+        toplevel_pkg = split_name[0]
+
+        finder = self._get_finder(fullname)
+
+        if finder is None:
+            return None
+        elif toplevel_pkg == 'ansible_collections':
+            return finder.find_spec(fullname, path=[self._pathctx])
+        else:
+            return finder.find_spec(fullname)
 
     def iter_modules(self, prefix):
         # NB: this currently represents only what's on disk, and does not handle package redirection
@@ -377,6 +423,23 @@ class _AnsibleCollectionPkgLoaderBase:
 
         return module_path, has_code, package_path
 
+    def exec_module(self, module):
+        # short-circuit redirect; avoid reinitializing existing modules
+        if self._redirect_module:
+            return
+
+        # execute the module's code in its namespace
+        code_obj = self.get_code(self._fullname)
+        if code_obj is not None:  # things like NS packages that can't have code on disk will return None
+            exec(code_obj, module.__dict__)
+
+    def create_module(self, spec):
+        # short-circuit redirect; we've already imported the redirected module, so just alias it and return it
+        if self._redirect_module:
+            return self._redirect_module
+        else:
+            return None
+
     def load_module(self, fullname):
         # short-circuit redirect; we've already imported the redirected module, so just alias it and return it
         if self._redirect_module:
@@ -531,12 +594,10 @@ class _AnsibleCollectionPkgLoader(_AnsibleCollectionPkgLoaderBase):
             # only search within the first collection we found
             self._subpackage_search_paths = [self._subpackage_search_paths[0]]
 
-    def load_module(self, fullname):
+    def _load_module(self, module):
         if not _meta_yml_to_dict:
             raise ValueError('ansible.utils.collection_loader._meta_yml_to_dict is not set')
 
-        module = super(_AnsibleCollectionPkgLoader, self).load_module(fullname)
-
         module._collection_meta = {}
         # TODO: load collection metadata, cache in __loader__ state
 
@@ -566,6 +627,17 @@ class _AnsibleCollectionPkgLoader(_AnsibleCollectionPkgLoaderBase):
 
         return module
 
+    def exec_module(self, module):
+        super(_AnsibleCollectionPkgLoader, self).exec_module(module)
+        self._load_module(module)
+
+    def create_module(self, spec):
+        return None
+
+    def load_module(self, fullname):
+        module = super(_AnsibleCollectionPkgLoader, self).load_module(fullname)
+        return self._load_module(module)
+
     def _canonicalize_meta(self, meta_dict):
         # TODO: rewrite import keys and all redirect targets that start with .. (current namespace) and . (current collection)
         # OR we could do it all on the fly?
@@ -588,7 +660,7 @@ class _AnsibleCollectionPkgLoader(_AnsibleCollectionPkgLoaderBase):
 # loads everything under a collection, including handling redirections defined by the collection
 class _AnsibleCollectionLoader(_AnsibleCollectionPkgLoaderBase):
     # HACK: stash this in a better place
-    _redirected_package_map = {}
+    _redirected_package_map = {}  # type: dict[str, str]
     _allows_package_code = True
 
     def _validate_args(self):
@@ -676,6 +748,17 @@ class _AnsibleInternalRedirectLoader:
         if not self._redirect:
             raise ImportError('not redirected, go ask path_hook')
 
+    def exec_module(self, module):
+        # should never see this
+        if not self._redirect:
+            raise ValueError('no redirect found for {0}'.format(module.__spec__.name))
+
+        # Replace the module with the redirect
+        sys.modules[module.__spec__.name] = import_module(self._redirect)
+
+    def create_module(self, spec):
+        return None
+
     def load_module(self, fullname):
         # since we're delegating to other loaders, this should only be called for internal redirects where we answered
         # find_module with this loader, in which case we'll just directly import the redirection target, insert it into
@@ -871,7 +954,7 @@ class AnsibleCollectionRef:
             return False
 
         return all(
-            # NOTE: keywords and identifiers are different in differnt Pythons
+            # NOTE: keywords and identifiers are different in different Pythons
             not iskeyword(ns_or_name) and is_python_identifier(ns_or_name)
             for ns_or_name in collection_name.split(u'.')
         )
diff --git a/lib/ansible/utils/collection_loader/_collection_meta.py b/lib/ansible/utils/collection_loader/_collection_meta.py
index 6baa9dca..deaac8e9 100644
--- a/lib/ansible/utils/collection_loader/_collection_meta.py
+++ b/lib/ansible/utils/collection_loader/_collection_meta.py
@@ -1,21 +1,16 @@
 # (c) 2019 Ansible Project
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
 
-# CAUTION: There are two implementations of the collection loader.
-#          They must be kept functionally identical, although their implementations may differ.
-#
-# 1) The controller implementation resides in the "lib/ansible/utils/collection_loader/" directory.
-#    It must function on all Python versions supported on the controller.
-# 2) The ansible-test implementation resides in the "test/lib/ansible_test/_util/target/legacy_collection_loader/" directory.
-#    It must function on all Python versions supported on managed hosts which are not supported by the controller.
+# CAUTION: This implementation of the collection loader is used by ansible-test.
+#          Because of this, it must be compatible with all Python versions supported on the controller or remote.
 
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 try:
-    from collections.abc import Mapping   # pylint: disable=ansible-bad-import-from
+    from collections.abc import Mapping
 except ImportError:
-    from collections import Mapping  # pylint: disable=ansible-bad-import-from
+    from collections import Mapping  # type: ignore[no-redef,attr-defined]  # pylint: disable=ansible-bad-import-from
 
 from ansible.module_utils.common.yaml import yaml_load
 
diff --git a/lib/ansible/utils/context_objects.py b/lib/ansible/utils/context_objects.py
index 71241749..efe15fea 100644
--- a/lib/ansible/utils/context_objects.py
+++ b/lib/ansible/utils/context_objects.py
@@ -10,8 +10,8 @@ Hold command line arguments for use in other modules
 """
 
 from abc import ABCMeta
+from collections.abc import Container, Mapping, Sequence, Set
 
-from ansible.module_utils.common._collections_compat import (Container, Mapping, Sequence, Set)
 from ansible.module_utils.common.collections import ImmutableDict
 from ansible.module_utils.six import add_metaclass, binary_type, text_type
 from ansible.utils.singleton import Singleton
diff --git a/lib/ansible/utils/display.py b/lib/ansible/utils/display.py
index 95cce2bb..b9d24654 100644
--- a/lib/ansible/utils/display.py
+++ b/lib/ansible/utils/display.py
@@ -36,18 +36,12 @@ from termios import TIOCGWINSZ
 
 from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleAssertionError
-from ansible.module_utils._text import to_bytes, to_text, to_native
-from ansible.module_utils.six import with_metaclass, text_type
+from ansible.module_utils._text import to_bytes, to_text
+from ansible.module_utils.six import text_type
 from ansible.utils.color import stringc
 from ansible.utils.singleton import Singleton
 from ansible.utils.unsafe_proxy import wrap_var
 
-try:
-    # Python 2
-    input = raw_input
-except NameError:
-    # Python 3, we already have raw_input
-    pass
 
 _LIBC = ctypes.cdll.LoadLibrary(ctypes.util.find_library('c'))
 # Set argtypes, to avoid segfault if the wrong type is provided,
@@ -204,7 +198,7 @@ b_COW_PATHS = (
 )
 
 
-class Display(with_metaclass(Singleton, object)):
+class Display(metaclass=Singleton):
 
     def __init__(self, verbosity=0):
 
@@ -225,7 +219,9 @@ class Display(with_metaclass(Singleton, object)):
             try:
                 cmd = subprocess.Popen([self.b_cowsay, "-l"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                 (out, err) = cmd.communicate()
-                self.cows_available = set([to_text(c) for c in out.split()])
+                if cmd.returncode:
+                    raise Exception
+                self.cows_available = {to_text(c) for c in out.split()}  # set comprehension
                 if C.ANSIBLE_COW_ACCEPTLIST and any(C.ANSIBLE_COW_ACCEPTLIST):
                     self.cows_available = set(C.ANSIBLE_COW_ACCEPTLIST).intersection(self.cows_available)
             except Exception:
@@ -268,11 +264,10 @@ class Display(with_metaclass(Singleton, object)):
                 msg2 = msg2 + u'\n'
 
             msg2 = to_bytes(msg2, encoding=self._output_encoding(stderr=stderr))
-            if sys.version_info >= (3,):
-                # Convert back to text string on python3
-                # We first convert to a byte string so that we get rid of
-                # characters that are invalid in the user's locale
-                msg2 = to_text(msg2, self._output_encoding(stderr=stderr), errors='replace')
+            # Convert back to text string
+            # We first convert to a byte string so that we get rid of
+            # characters that are invalid in the user's locale
+            msg2 = to_text(msg2, self._output_encoding(stderr=stderr), errors='replace')
 
             # Note: After Display() class is refactored need to update the log capture
             # code in 'bin/ansible-connection' (and other relevant places).
@@ -296,9 +291,8 @@ class Display(with_metaclass(Singleton, object)):
             # color and characters that are invalid in the user's locale
             msg2 = to_bytes(nocolor.lstrip(u'\n'))
 
-            if sys.version_info >= (3,):
-                # Convert back to text string on python3
-                msg2 = to_text(msg2, self._output_encoding(stderr=stderr))
+            # Convert back to text string
+            msg2 = to_text(msg2, self._output_encoding(stderr=stderr))
 
             lvl = logging.INFO
             if color:
@@ -467,10 +461,9 @@ class Display(with_metaclass(Singleton, object)):
     @staticmethod
     def prompt(msg, private=False):
         prompt_string = to_bytes(msg, encoding=Display._output_encoding())
-        if sys.version_info >= (3,):
-            # Convert back into text on python3.  We do this double conversion
-            # to get rid of characters that are illegal in the user's locale
-            prompt_string = to_text(prompt_string)
+        # Convert back into text.  We do this double conversion
+        # to get rid of characters that are illegal in the user's locale
+        prompt_string = to_text(prompt_string)
 
         if private:
             return getpass.getpass(prompt_string)
diff --git a/lib/ansible/utils/galaxy.py b/lib/ansible/utils/galaxy.py
index cb1f125b..bbb26fb1 100644
--- a/lib/ansible/utils/galaxy.py
+++ b/lib/ansible/utils/galaxy.py
@@ -25,6 +25,7 @@ from subprocess import Popen, PIPE
 import tarfile
 
 import ansible.constants as C
+from ansible import context
 from ansible.errors import AnsibleError
 from ansible.utils.display import Display
 from ansible.module_utils.common.process import get_bin_path
@@ -62,7 +63,19 @@ def scm_archive_resource(src, scm='git', name=None, version='HEAD', keep_scm_met
         raise AnsibleError("could not find/use %s, it is required to continue with installing %s" % (scm, src))
 
     tempdir = tempfile.mkdtemp(dir=C.DEFAULT_LOCAL_TMP)
-    clone_cmd = [scm_path, 'clone', src, name]
+    clone_cmd = [scm_path, 'clone']
+
+    # Add specific options for ignoring certificates if requested
+    ignore_certs = context.CLIARGS['ignore_certs']
+
+    if ignore_certs:
+        if scm == 'git':
+            clone_cmd.extend(['-c', 'http.sslVerify=false'])
+        elif scm == 'hg':
+            clone_cmd.append('--insecure')
+
+    clone_cmd.extend([src, name])
+
     run_scm_cmd(clone_cmd, tempdir)
 
     if scm == 'git' and version:
diff --git a/lib/ansible/utils/hashing.py b/lib/ansible/utils/hashing.py
index 5f36522e..71300d61 100644
--- a/lib/ansible/utils/hashing.py
+++ b/lib/ansible/utils/hashing.py
@@ -21,22 +21,13 @@ __metaclass__ = type
 
 import os
 
-# Note, sha1 is the only hash algorithm compatible with python2.4 and with
-# FIPS-140 mode (as of 11-2014)
-try:
-    from hashlib import sha1
-except ImportError:
-    from sha import sha as sha1
+from hashlib import sha1
 
-# Backwards compat only
 try:
     from hashlib import md5 as _md5
 except ImportError:
-    try:
-        from md5 import md5 as _md5
-    except ImportError:
-        # Assume we're running in FIPS mode here
-        _md5 = None
+    # Assume we're running in FIPS mode here
+    _md5 = None
 
 from ansible.errors import AnsibleError
 from ansible.module_utils._text import to_bytes
diff --git a/lib/ansible/utils/jsonrpc.py b/lib/ansible/utils/jsonrpc.py
index e48c979d..8d5b0f6c 100644
--- a/lib/ansible/utils/jsonrpc.py
+++ b/lib/ansible/utils/jsonrpc.py
@@ -5,12 +5,12 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import json
+import pickle
 import traceback
 
 from ansible.module_utils._text import to_text
 from ansible.module_utils.connection import ConnectionError
 from ansible.module_utils.six import binary_type, text_type
-from ansible.module_utils.six.moves import cPickle
 from ansible.utils.display import Display
 
 display = Display()
@@ -18,7 +18,7 @@ display = Display()
 
 class JsonRpcServer(object):
 
-    _objects = set()
+    _objects = set()  # type: set[object]
 
     def handle_request(self, request):
         request = json.loads(to_text(request, errors='surrogate_then_replace'))
@@ -84,7 +84,7 @@ class JsonRpcServer(object):
             result = to_text(result)
         if not isinstance(result, text_type):
             response["result_type"] = "pickle"
-            result = to_text(cPickle.dumps(result, protocol=0))
+            result = to_text(pickle.dumps(result, protocol=0))
         response['result'] = result
         return response
 
diff --git a/lib/ansible/utils/listify.py b/lib/ansible/utils/listify.py
index 89a6583c..4f2ae9d4 100644
--- a/lib/ansible/utils/listify.py
+++ b/lib/ansible/utils/listify.py
@@ -19,8 +19,9 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+from collections.abc import Iterable
+
 from ansible.module_utils.six import string_types
-from ansible.module_utils.common._collections_compat import Iterable
 
 
 __all__ = ['listify_lookup_plugin_terms']
diff --git a/lib/ansible/utils/multiprocessing.py b/lib/ansible/utils/multiprocessing.py
index dcc18659..2912f712 100644
--- a/lib/ansible/utils/multiprocessing.py
+++ b/lib/ansible/utils/multiprocessing.py
@@ -14,8 +14,4 @@ import multiprocessing
 #
 # This exists in utils to allow it to be easily imported into various places
 # without causing circular import or dependency problems
-try:
-    context = multiprocessing.get_context('fork')
-except AttributeError:
-    # Py2 has no context functionality, and only supports fork
-    context = multiprocessing
+context = multiprocessing.get_context('fork')
diff --git a/lib/ansible/utils/plugin_docs.py b/lib/ansible/utils/plugin_docs.py
index eb03dc08..3499800c 100644
--- a/lib/ansible/utils/plugin_docs.py
+++ b/lib/ansible/utils/plugin_docs.py
@@ -4,12 +4,13 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+from collections.abc import MutableMapping, MutableSet, MutableSequence
+
 from ansible import constants as C
 from ansible.release import __version__ as ansible_version
 from ansible.errors import AnsibleError
 from ansible.module_utils.six import string_types
 from ansible.module_utils._text import to_native
-from ansible.module_utils.common._collections_compat import MutableMapping, MutableSet, MutableSequence
 from ansible.parsing.plugin_docs import read_docstring
 from ansible.parsing.yaml.loader import AnsibleLoader
 from ansible.utils.display import Display
@@ -37,7 +38,7 @@ def merge_fragment(target, source):
             elif isinstance(target[key], MutableSequence):
                 value = sorted(frozenset(value + target[key]))
             else:
-                raise Exception("Attempt to extend a documentation fragement, invalid type for %s" % key)
+                raise Exception("Attempt to extend a documentation fragment, invalid type for %s" % key)
         target[key] = value
 
 
diff --git a/lib/ansible/utils/py3compat.py b/lib/ansible/utils/py3compat.py
index 6b46b5e8..88d9fdff 100644
--- a/lib/ansible/utils/py3compat.py
+++ b/lib/ansible/utils/py3compat.py
@@ -14,9 +14,10 @@ __metaclass__ = type
 import os
 import sys
 
+from collections.abc import MutableMapping
+
 from ansible.module_utils.six import PY3
 from ansible.module_utils._text import to_bytes, to_text
-from ansible.module_utils.common._collections_compat import MutableMapping
 
 __all__ = ('environ',)
 
diff --git a/lib/ansible/utils/ssh_functions.py b/lib/ansible/utils/ssh_functions.py
index ec8984d5..a728889a 100644
--- a/lib/ansible/utils/ssh_functions.py
+++ b/lib/ansible/utils/ssh_functions.py
@@ -27,7 +27,7 @@ from ansible.module_utils._text import to_bytes
 from ansible.module_utils.compat.paramiko import paramiko
 
 
-_HAS_CONTROLPERSIST = {}
+_HAS_CONTROLPERSIST = {}  # type: dict[str, bool]
 
 
 def check_for_controlpersist(ssh_executable):
diff --git a/lib/ansible/utils/unsafe_proxy.py b/lib/ansible/utils/unsafe_proxy.py
index 7ed3cffc..e10fa8a0 100644
--- a/lib/ansible/utils/unsafe_proxy.py
+++ b/lib/ansible/utils/unsafe_proxy.py
@@ -53,8 +53,9 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
+from collections.abc import Mapping, Set
+
 from ansible.module_utils._text import to_bytes, to_text
-from ansible.module_utils.common._collections_compat import Mapping, Set
 from ansible.module_utils.common.collections import is_sequence
 from ansible.module_utils.six import string_types, binary_type, text_type
 from ansible.utils.native_jinja import NativeJinjaText
diff --git a/lib/ansible/utils/vars.py b/lib/ansible/utils/vars.py
index 7545f1d4..74bf1425 100644
--- a/lib/ansible/utils/vars.py
+++ b/lib/ansible/utils/vars.py
@@ -23,15 +23,14 @@ import keyword
 import random
 import uuid
 
+from collections.abc import MutableMapping, MutableSequence
 from json import dumps
 
-
 from ansible import constants as C
 from ansible import context
 from ansible.errors import AnsibleError, AnsibleOptionsError
-from ansible.module_utils.six import iteritems, string_types, PY3
+from ansible.module_utils.six import string_types, PY3
 from ansible.module_utils._text import to_native, to_text
-from ansible.module_utils.common._collections_compat import MutableMapping, MutableSequence
 from ansible.parsing.splitter import parse_kv
 
 
@@ -129,7 +128,7 @@ def merge_hash(x, y, recursive=True, list_merge='replace'):
     # there is a high probability x will be the "default" dict the user
     # want to "patch" with y
     # therefore x will have much more elements than y
-    for key, y_value in iteritems(y):
+    for key, y_value in y.items():
         # if `key` isn't in x
         # update x and move on to the next element of y
         if key not in x: