New upstream version 2.14.1

16 files changed, 369 insertions, 8 deletions

diff --git a/test/integration/targets/ansible-galaxy-role/tasks/main.yml b/test/integration/targets/ansible-galaxy-role/tasks/main.yml
index e49e4e29..03d0b3c2 100644
--- a/test/integration/targets/ansible-galaxy-role/tasks/main.yml
+++ b/test/integration/targets/ansible-galaxy-role/tasks/main.yml
@@ -1,3 +1,6 @@
+- name: Install role from Galaxy (should not fail with AttributeError)
+  command: ansible-galaxy role install ansible.nope -vvvv --ignore-errors
+
 - name: Archive directories
   file:
     state: directory
diff --git a/test/integration/targets/ansible-galaxy/runme.sh b/test/integration/targets/ansible-galaxy/runme.sh
index 4005531a..7d966e29 100755
--- a/test/integration/targets/ansible-galaxy/runme.sh
+++ b/test/integration/targets/ansible-galaxy/runme.sh
@@ -103,7 +103,11 @@ f_ansible_galaxy_status "install of local git repo"
 mkdir -p "${galaxy_testdir}"
 pushd "${galaxy_testdir}"
 
-    ansible-galaxy install git+file:///"${galaxy_local_test_role_git_repo}" "$@"
+    # minimum verbosity is hardcoded to include calls to Galaxy
+    ansible-galaxy install git+file:///"${galaxy_local_test_role_git_repo}" "$@" -vvvv 2>&1 | tee out.txt
+
+    # Test no initial call is made to Galaxy
+    grep out.txt -e "https://galaxy.ansible.com" && cat out.txt && exit 1
 
     # Test that the role was installed to the expected directory
     [[ -d "${HOME}/.ansible/roles/${galaxy_local_test_role}" ]]
diff --git a/test/integration/targets/copy/tasks/main.yml b/test/integration/targets/copy/tasks/main.yml
index 817fe0a1..b86c56ac 100644
--- a/test/integration/targets/copy/tasks/main.yml
+++ b/test/integration/targets/copy/tasks/main.yml
@@ -96,6 +96,9 @@
           - 'diff_output.diff[0].before == ""'
           - '"Ansible managed" in diff_output.diff[0].after'
 
+    - name: tests with remote_src and non files
+      import_tasks: src_remote_file_is_not_file.yml
+
   always:
     - name: Cleaning
       file:
diff --git a/test/integration/targets/copy/tasks/src_remote_file_is_not_file.yml b/test/integration/targets/copy/tasks/src_remote_file_is_not_file.yml
new file mode 100644
index 00000000..2cda7d37
--- /dev/null
+++ b/test/integration/targets/copy/tasks/src_remote_file_is_not_file.yml
@@ -0,0 +1,39 @@
+- name: test remote src non files
+  vars:
+    destfile: '{{remote_dir}}/whocares'
+  block:
+  - name: mess with dev/null
+    copy:
+      src: /dev/null
+      dest: "{{destfile}}"
+      remote_src: true
+    become: true
+    register: dev_null_fail
+    ignore_errors: true
+
+  - name: ensure we failed
+    assert:
+      that:
+        - dev_null_fail is failed
+        - "'not a file' in dev_null_fail.msg"
+
+  - name: now with file existing
+    file: state=touch path="{{destfile}}"
+
+  - name: mess with dev/null again
+    copy:
+      src: /dev/null
+      dest: "{{destfile}}"
+      remote_src: true
+    become: true
+    register: dev_null_fail
+    ignore_errors: true
+
+  - name: ensure we failed, again
+    assert:
+      that:
+        - dev_null_fail is failed
+        - "'not a file' in dev_null_fail.msg"
+  always:
+  - name: cleanup
+    file: state=absent path="{{destfile}}"
diff --git a/test/integration/targets/fork_safe_stdio/aliases b/test/integration/targets/fork_safe_stdio/aliases
new file mode 100644
index 00000000..e968db72
--- /dev/null
+++ b/test/integration/targets/fork_safe_stdio/aliases
@@ -0,0 +1,3 @@
+shippable/posix/group3
+context/controller
+skip/macos
diff --git a/test/integration/targets/fork_safe_stdio/callback_plugins/spewstdio.py b/test/integration/targets/fork_safe_stdio/callback_plugins/spewstdio.py
new file mode 100644
index 00000000..6ed6ef34
--- /dev/null
+++ b/test/integration/targets/fork_safe_stdio/callback_plugins/spewstdio.py
@@ -0,0 +1,58 @@
+import atexit
+import os
+import sys
+
+from ansible.plugins.callback import CallbackBase
+from ansible.utils.display import Display
+from threading import Thread
+
+# This callback plugin reliably triggers the deadlock from https://github.com/ansible/ansible-runner/issues/1164 when
+# run on a TTY/PTY. It starts a thread in the controller that spews unprintable characters to stdout as fast as
+# possible, while causing forked children to write directly to the inherited stdout immediately post-fork. If a fork
+# occurs while the spew thread holds stdout's internal BufferedIOWriter lock, the lock will be orphaned in the child,
+# and attempts to write to stdout there will hang forever.
+
+# Any mechanism that ensures non-main threads do not hold locks before forking should allow this test to pass.
+
+# ref: https://docs.python.org/3/library/io.html#multi-threading
+# ref: https://github.com/python/cpython/blob/0547a981ae413248b21a6bb0cb62dda7d236fe45/Modules/_io/bufferedio.c#L268
+
+
+class CallbackModule(CallbackBase):
+    CALLBACK_VERSION = 2.0
+    CALLBACK_NAME = 'spewstdio'
+
+    def __init__(self):
+        super().__init__()
+        self.display = Display()
+
+        if os.environ.get('SPEWSTDIO_ENABLED', '0') != '1':
+            self.display.warning('spewstdio test plugin loaded but disabled; set SPEWSTDIO_ENABLED=1 to enable')
+            return
+
+        self.display = Display()
+        self._keep_spewing = True
+
+        # cause the child to write directly to stdout immediately post-fork
+        os.register_at_fork(after_in_child=lambda: print(f"hi from forked child pid {os.getpid()}"))
+
+        # in passing cases, stop spewing when the controller is exiting to prevent fatal errors on final flush
+        atexit.register(self.stop_spew)
+
+        self._spew_thread = Thread(target=self.spew, daemon=True)
+        self._spew_thread.start()
+
+    def stop_spew(self):
+        self._keep_spewing = False
+
+    def spew(self):
+        # dump a message so we know the callback thread has started
+        self.display.warning("spewstdio STARTING NONPRINTING SPEW ON BACKGROUND THREAD")
+
+        while self._keep_spewing:
+            # dump a non-printing control character directly to stdout to avoid junking up the screen while still
+            # doing lots of writes and flushes.
+            sys.stdout.write('\x1b[K')
+            sys.stdout.flush()
+
+        self.display.warning("spewstdio STOPPING SPEW")
diff --git a/test/integration/targets/fork_safe_stdio/hosts b/test/integration/targets/fork_safe_stdio/hosts
new file mode 100644
index 00000000..675e82ad
--- /dev/null
+++ b/test/integration/targets/fork_safe_stdio/hosts
@@ -0,0 +1,5 @@
+[all]
+local-[1:10]
+
+[all:vars]
+ansible_connection=local
diff --git a/test/integration/targets/fork_safe_stdio/run-with-pty.py b/test/integration/targets/fork_safe_stdio/run-with-pty.py
new file mode 100755
index 00000000..46391528
--- /dev/null
+++ b/test/integration/targets/fork_safe_stdio/run-with-pty.py
@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+"""Run a command using a PTY."""
+
+import sys
+
+if sys.version_info < (3, 10):
+    import vendored_pty as pty
+else:
+    import pty
+
+sys.exit(1 if pty.spawn(sys.argv[1:]) else 0)
diff --git a/test/integration/targets/fork_safe_stdio/runme.sh b/test/integration/targets/fork_safe_stdio/runme.sh
new file mode 100755
index 00000000..4438c3fe
--- /dev/null
+++ b/test/integration/targets/fork_safe_stdio/runme.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -eu
+
+echo "testing for stdio deadlock on forked workers (10s timeout)..."
+
+# Enable a callback that trips deadlocks on forked-child stdout, time out after 10s; forces running
+# in a pty, since that tends to be much slower than raw file I/O and thus more likely to trigger the deadlock.
+# Redirect stdout to /dev/null since it's full of non-printable garbage we don't want to display unless it failed
+ANSIBLE_CALLBACKS_ENABLED=spewstdio SPEWSTDIO_ENABLED=1 python run-with-pty.py timeout 10s ansible-playbook -i hosts -f 5 test.yml > stdout.txt && RC=$? || RC=$?
+
+if [ $RC != 0 ]; then
+  echo "failed; likely stdout deadlock. dumping raw output (may be very large)"
+  cat stdout.txt
+  exit 1
+fi
+
+grep -q -e "spewstdio STARTING NONPRINTING SPEW ON BACKGROUND THREAD" stdout.txt || (echo "spewstdio callback was not enabled"; exit 1)
+
+echo "PASS"
diff --git a/test/integration/targets/fork_safe_stdio/test.yml b/test/integration/targets/fork_safe_stdio/test.yml
new file mode 100644
index 00000000..d60f0715
--- /dev/null
+++ b/test/integration/targets/fork_safe_stdio/test.yml
@@ -0,0 +1,5 @@
+- hosts: all
+  gather_facts: no
+  tasks:
+  - debug:
+      msg: yo
diff --git a/test/integration/targets/fork_safe_stdio/vendored_pty.py b/test/integration/targets/fork_safe_stdio/vendored_pty.py
new file mode 100644
index 00000000..bc70803b
--- /dev/null
+++ b/test/integration/targets/fork_safe_stdio/vendored_pty.py
@@ -0,0 +1,189 @@
+# Vendored copy of https://github.com/python/cpython/blob/3680ebed7f3e529d01996dd0318601f9f0d02b4b/Lib/pty.py
+# PSF License (see licenses/PSF-license.txt or https://opensource.org/licenses/Python-2.0)
+"""Pseudo terminal utilities."""
+
+# Bugs: No signal handling.  Doesn't set slave termios and window size.
+#       Only tested on Linux, FreeBSD, and macOS.
+# See:  W. Richard Stevens. 1992.  Advanced Programming in the
+#       UNIX Environment.  Chapter 19.
+# Author: Steen Lumholt -- with additions by Guido.
+
+from select import select
+import os
+import sys
+import tty
+
+# names imported directly for test mocking purposes
+from os import close, waitpid
+from tty import setraw, tcgetattr, tcsetattr
+
+__all__ = ["openpty", "fork", "spawn"]
+
+STDIN_FILENO = 0
+STDOUT_FILENO = 1
+STDERR_FILENO = 2
+
+CHILD = 0
+
+def openpty():
+    """openpty() -> (master_fd, slave_fd)
+    Open a pty master/slave pair, using os.openpty() if possible."""
+
+    try:
+        return os.openpty()
+    except (AttributeError, OSError):
+        pass
+    master_fd, slave_name = _open_terminal()
+    slave_fd = slave_open(slave_name)
+    return master_fd, slave_fd
+
+def master_open():
+    """master_open() -> (master_fd, slave_name)
+    Open a pty master and return the fd, and the filename of the slave end.
+    Deprecated, use openpty() instead."""
+
+    try:
+        master_fd, slave_fd = os.openpty()
+    except (AttributeError, OSError):
+        pass
+    else:
+        slave_name = os.ttyname(slave_fd)
+        os.close(slave_fd)
+        return master_fd, slave_name
+
+    return _open_terminal()
+
+def _open_terminal():
+    """Open pty master and return (master_fd, tty_name)."""
+    for x in 'pqrstuvwxyzPQRST':
+        for y in '0123456789abcdef':
+            pty_name = '/dev/pty' + x + y
+            try:
+                fd = os.open(pty_name, os.O_RDWR)
+            except OSError:
+                continue
+            return (fd, '/dev/tty' + x + y)
+    raise OSError('out of pty devices')
+
+def slave_open(tty_name):
+    """slave_open(tty_name) -> slave_fd
+    Open the pty slave and acquire the controlling terminal, returning
+    opened filedescriptor.
+    Deprecated, use openpty() instead."""
+
+    result = os.open(tty_name, os.O_RDWR)
+    try:
+        from fcntl import ioctl, I_PUSH
+    except ImportError:
+        return result
+    try:
+        ioctl(result, I_PUSH, "ptem")
+        ioctl(result, I_PUSH, "ldterm")
+    except OSError:
+        pass
+    return result
+
+def fork():
+    """fork() -> (pid, master_fd)
+    Fork and make the child a session leader with a controlling terminal."""
+
+    try:
+        pid, fd = os.forkpty()
+    except (AttributeError, OSError):
+        pass
+    else:
+        if pid == CHILD:
+            try:
+                os.setsid()
+            except OSError:
+                # os.forkpty() already set us session leader
+                pass
+        return pid, fd
+
+    master_fd, slave_fd = openpty()
+    pid = os.fork()
+    if pid == CHILD:
+        # Establish a new session.
+        os.setsid()
+        os.close(master_fd)
+
+        # Slave becomes stdin/stdout/stderr of child.
+        os.dup2(slave_fd, STDIN_FILENO)
+        os.dup2(slave_fd, STDOUT_FILENO)
+        os.dup2(slave_fd, STDERR_FILENO)
+        if slave_fd > STDERR_FILENO:
+            os.close(slave_fd)
+
+        # Explicitly open the tty to make it become a controlling tty.
+        tmp_fd = os.open(os.ttyname(STDOUT_FILENO), os.O_RDWR)
+        os.close(tmp_fd)
+    else:
+        os.close(slave_fd)
+
+    # Parent and child process.
+    return pid, master_fd
+
+def _writen(fd, data):
+    """Write all the data to a descriptor."""
+    while data:
+        n = os.write(fd, data)
+        data = data[n:]
+
+def _read(fd):
+    """Default read function."""
+    return os.read(fd, 1024)
+
+def _copy(master_fd, master_read=_read, stdin_read=_read):
+    """Parent copy loop.
+    Copies
+            pty master -> standard output   (master_read)
+            standard input -> pty master    (stdin_read)"""
+    fds = [master_fd, STDIN_FILENO]
+    while fds:
+        rfds, _wfds, _xfds = select(fds, [], [])
+
+        if master_fd in rfds:
+            # Some OSes signal EOF by returning an empty byte string,
+            # some throw OSErrors.
+            try:
+                data = master_read(master_fd)
+            except OSError:
+                data = b""
+            if not data:  # Reached EOF.
+                return    # Assume the child process has exited and is
+                          # unreachable, so we clean up.
+            else:
+                os.write(STDOUT_FILENO, data)
+
+        if STDIN_FILENO in rfds:
+            data = stdin_read(STDIN_FILENO)
+            if not data:
+                fds.remove(STDIN_FILENO)
+            else:
+                _writen(master_fd, data)
+
+def spawn(argv, master_read=_read, stdin_read=_read):
+    """Create a spawned process."""
+    if isinstance(argv, str):
+        argv = (argv,)
+    sys.audit('pty.spawn', argv)
+
+    pid, master_fd = fork()
+    if pid == CHILD:
+        os.execlp(argv[0], *argv)
+
+    try:
+        mode = tcgetattr(STDIN_FILENO)
+        setraw(STDIN_FILENO)
+        restore = True
+    except tty.error:    # This is the same as termios.error
+        restore = False
+
+    try:
+        _copy(master_fd, master_read, stdin_read)
+    finally:
+        if restore:
+            tcsetattr(STDIN_FILENO, tty.TCSAFLUSH, mode)
+
+    close(master_fd)
+    return waitpid(pid, 0)[1]
diff --git a/test/integration/targets/jinja2_native_types/runme.sh b/test/integration/targets/jinja2_native_types/runme.sh
index f648f875..a6c2befa 100755
--- a/test/integration/targets/jinja2_native_types/runme.sh
+++ b/test/integration/targets/jinja2_native_types/runme.sh
@@ -7,4 +7,5 @@ ansible-playbook runtests.yml -v "$@"
 ansible-playbook --vault-password-file test_vault_pass test_vault.yml -v "$@"
 ansible-playbook test_hostvars.yml -v "$@"
 ansible-playbook nested_undefined.yml -v "$@"
+ansible-playbook test_preserving_quotes.yml -v "$@"
 unset ANSIBLE_JINJA2_NATIVE
diff --git a/test/integration/targets/jinja2_native_types/test_casting.yml b/test/integration/targets/jinja2_native_types/test_casting.yml
index 8627a056..5e9c76d6 100644
--- a/test/integration/targets/jinja2_native_types/test_casting.yml
+++ b/test/integration/targets/jinja2_native_types/test_casting.yml
@@ -13,7 +13,7 @@
 
 - assert:
     that:
-        - 'int_to_str == "2"'
+        - int_to_str == "'2'"
         - 'int_to_str|type_debug in ["str", "unicode"]'
         - 'int_to_str2 == "2"'
         - 'int_to_str2|type_debug in ["NativeJinjaText"]'
diff --git a/test/integration/targets/jinja2_native_types/test_concatentation.yml b/test/integration/targets/jinja2_native_types/test_concatentation.yml
index 8a8077b6..24a90381 100644
--- a/test/integration/targets/jinja2_native_types/test_concatentation.yml
+++ b/test/integration/targets/jinja2_native_types/test_concatentation.yml
@@ -22,7 +22,7 @@
 
 - assert:
     that:
-        - 'string_sum == "12"'
+        - string_sum == "'12'"
         - 'string_sum|type_debug in ["str", "unicode"]'
 
 - name: add two lists
diff --git a/test/integration/targets/jinja2_native_types/test_preserving_quotes.yml b/test/integration/targets/jinja2_native_types/test_preserving_quotes.yml
new file mode 100644
index 00000000..d6fea10e
--- /dev/null
+++ b/test/integration/targets/jinja2_native_types/test_preserving_quotes.yml
@@ -0,0 +1,14 @@
+- hosts: localhost
+  gather_facts: false
+  tasks:
+    - assert:
+        that:
+          - quoted_str == '"hello"'
+          - empty_quoted_str == '""'
+      vars:
+        third_nested_lvl: '"hello"'
+        second_nested_lvl: "{{ third_nested_lvl }}"
+        first_nested_lvl: "{{ second_nested_lvl }}"
+        quoted_str: "{{ first_nested_lvl }}"
+        empty_quoted_str: "{{ empty_str }}"
+        empty_str: '""'
diff --git a/test/integration/targets/jinja_plugins/tasks/main.yml b/test/integration/targets/jinja_plugins/tasks/main.yml
index 824e8306..d3d6e2e8 100644
--- a/test/integration/targets/jinja_plugins/tasks/main.yml
+++ b/test/integration/targets/jinja_plugins/tasks/main.yml
@@ -1,4 +1,6 @@
 - shell: ansible-playbook {{ verbosity }} playbook.yml
+  environment:
+    ANSIBLE_FORCE_COLOR: no
   args:
     chdir: '{{ role_path }}'
   vars:
@@ -8,10 +10,14 @@
 - debug:
     var: result
 
+- set_fact:
+    # NOTE: This will cram words together that were manually wrapped, which should be OK for this test.
+    stderr: "{{ result.stderr | replace('\n', '') }}"
+
 - assert:
     that:
-      - '"[WARNING]: Skipping filter plugin" in result.stderr'
-      - '"[WARNING]: Skipping test plugin" in result.stderr'
-      - result.stderr|regex_findall('bad_collection_filter')|length == 3
-      - result.stderr|regex_findall('bad_collection_filter2')|length == 1
-      - result.stderr|regex_findall('bad_collection_test')|length == 2
+      - '"[WARNING]: Skipping filter plugin" in stderr'
+      - '"[WARNING]: Skipping test plugin" in stderr'
+      - stderr|regex_findall('bad_collection_filter')|length == 3
+      - stderr|regex_findall('bad_collection_filter2')|length == 1
+      - stderr|regex_findall('bad_collection_test')|length == 2