diff options
| author | Lee Garrett <lgarrett@rocketjump.eu> | 2022-08-25 04:18:50 +0200 |
|---|---|---|
| committer | Lee Garrett <lgarrett@rocketjump.eu> | 2022-08-25 04:18:50 +0200 |
| commit | 5883937d823fe68e35dbedf2a9d45ecaf6636470 (patch) | |
| tree | 6ca8e5156b2bc203b16339d8c81a5de4a26fc2cb /test | |
| parent | df2a2cd18c338647061f3448248f8b97b6971f49 (diff) | |
| download | debian-ansible-core-5883937d823fe68e35dbedf2a9d45ecaf6636470.zip | |
New upstream version 2.13.3
Diffstat (limited to 'test')
244 files changed, 2653 insertions, 649 deletions
diff --git a/test/integration/targets/ansible-galaxy-collection-scm/tasks/main.yml b/test/integration/targets/ansible-galaxy-collection-scm/tasks/main.yml index 986da2f9..546c4083 100644 --- a/test/integration/targets/ansible-galaxy-collection-scm/tasks/main.yml +++ b/test/integration/targets/ansible-galaxy-collection-scm/tasks/main.yml @@ -23,6 +23,10 @@ - include_tasks: ./multi_collection_repo_individual.yml - include_tasks: ./setup_recursive_scm_dependency.yml - include_tasks: ./scm_dependency_deduplication.yml + - include_tasks: ./test_supported_resolvelib_versions.yml + loop: "{{ supported_resolvelib_versions }}" + loop_control: + loop_var: resolvelib_version - include_tasks: ./download.yml - include_tasks: ./setup_collection_bad_version.yml - include_tasks: ./test_invalid_version.yml diff --git a/test/integration/targets/ansible-galaxy-collection-scm/tasks/requirements.yml b/test/integration/targets/ansible-galaxy-collection-scm/tasks/requirements.yml index c7743426..10070f1a 100644 --- a/test/integration/targets/ansible-galaxy-collection-scm/tasks/requirements.yml +++ b/test/integration/targets/ansible-galaxy-collection-scm/tasks/requirements.yml @@ -30,7 +30,7 @@ nor 'source' point to a concrete resolvable collection artifact. Also 'name' is not an FQCN. A valid collection name must be in the format <namespace>.<collection>. Please make sure that the - namespace and the collection name contain characters from + namespace and the collection name contain characters from [a-zA-Z0-9_] only." in result.stderr - name: test source is not a git repo even if name is provided diff --git a/test/integration/targets/ansible-galaxy-collection-scm/tasks/test_supported_resolvelib_versions.yml b/test/integration/targets/ansible-galaxy-collection-scm/tasks/test_supported_resolvelib_versions.yml new file mode 100644 index 00000000..029cbb3a --- /dev/null +++ b/test/integration/targets/ansible-galaxy-collection-scm/tasks/test_supported_resolvelib_versions.yml @@ -0,0 +1,25 @@ +- vars: + venv_cmd: "{{ ansible_python_interpreter ~ ' -m venv' }}" + venv_dest: "{{ galaxy_dir }}/test_venv_{{ resolvelib_version }}" + block: + - name: install another version of resolvelib that is supported by ansible-galaxy + pip: + name: resolvelib + version: "{{ resolvelib_version }}" + state: present + virtualenv_command: "{{ venv_cmd }}" + virtualenv: "{{ venv_dest }}" + virtualenv_site_packages: True + + - include_tasks: ./scm_dependency_deduplication.yml + args: + apply: + environment: + PATH: "{{ venv_dest }}/bin:{{ ansible_env.PATH }}" + ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg' + + always: + - name: remove test venv + file: + path: "{{ venv_dest }}" + state: absent diff --git a/test/integration/targets/ansible-galaxy-collection-scm/vars/main.yml b/test/integration/targets/ansible-galaxy-collection-scm/vars/main.yml index a82f25dc..cd198c64 100644 --- a/test/integration/targets/ansible-galaxy-collection-scm/vars/main.yml +++ b/test/integration/targets/ansible-galaxy-collection-scm/vars/main.yml @@ -3,3 +3,9 @@ alt_install_path: "{{ galaxy_dir }}/other_collections/ansible_collections" scm_path: "{{ galaxy_dir }}/development" test_repo_path: "{{ galaxy_dir }}/development/ansible_test" test_error_repo_path: "{{ galaxy_dir }}/development/error_test" + +supported_resolvelib_versions: + - "0.5.3" # Oldest supported + - "0.6.0" + - "0.7.0" + - "0.8.0" diff --git a/test/integration/targets/ansible-galaxy-collection/library/setup_collections.py b/test/integration/targets/ansible-galaxy-collection/library/setup_collections.py index adefba05..58ec9b2a 100644 --- a/test/integration/targets/ansible-galaxy-collection/library/setup_collections.py +++ b/test/integration/targets/ansible-galaxy-collection/library/setup_collections.py @@ -182,8 +182,6 @@ def sign_manifest(signature_path, manifest_path, module, collection_setup_result "--pinentry-mode", "loopback", "--yes", - "--passphrase", - "SECRET", "--homedir", module.params['signature_dir'], "--detach-sign", diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/download.yml b/test/integration/targets/ansible-galaxy-collection/tasks/download.yml index e00d0b83..b651a73e 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/download.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/download.yml @@ -169,3 +169,8 @@ that: - '"Downloading collection ''ansible_test.my_collection:1.0.0'' to" in download_collection.stdout' - download_collection_actual.stat.exists + +- name: remove test download dir + file: + path: '{{ galaxy_dir }}/download' + state: absent diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/fail_fast_resolvelib.yml b/test/integration/targets/ansible-galaxy-collection/tasks/fail_fast_resolvelib.yml new file mode 100644 index 00000000..eb471f8e --- /dev/null +++ b/test/integration/targets/ansible-galaxy-collection/tasks/fail_fast_resolvelib.yml @@ -0,0 +1,45 @@ +# resolvelib>=0.6.0 added an 'incompatibilities' parameter to find_matches +# If incompatibilities aren't removed from the viable candidates, this example causes infinite resursion +- name: test resolvelib removes incompatibilites in find_matches and errors quickly (prevent infinite recursion) + block: + - name: create collection dir + file: + dest: "{{ galaxy_dir }}/resolvelib/ns/coll" + state: directory + + - name: create galaxy.yml with a dependecy on a galaxy-sourced collection + copy: + dest: "{{ galaxy_dir }}/resolvelib/ns/coll/galaxy.yml" + content: | + namespace: ns + name: coll + authors: + - ansible-core + readme: README.md + version: "1.0.0" + dependencies: + namespace1.name1: "0.0.5" + + - name: build the collection + command: ansible-galaxy collection build ns/coll + args: + chdir: "{{ galaxy_dir }}/resolvelib" + + - name: install a conflicting version of the dep with the tarfile (expected failure) + command: ansible-galaxy collection install namespace1.name1:1.0.9 ns-coll-1.0.0.tar.gz -vvvvv -s {{ test_name }} -p collections/ + args: + chdir: "{{ galaxy_dir }}/resolvelib" + timeout: 30 + ignore_errors: yes + register: incompatible + + - assert: + that: + - incompatible.failed + - not incompatible.msg.startswith("The command action failed to execute in the expected time frame") + + always: + - name: cleanup resolvelib test + file: + dest: "{{ galaxy_dir }}/resolvelib" + state: absent diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/init.yml b/test/integration/targets/ansible-galaxy-collection/tasks/init.yml index a57cafc3..85a87575 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/init.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/init.yml @@ -43,6 +43,51 @@ - (init_custom_path_actual.files | map(attribute='path') | list)[1] | basename in ['docs', 'plugins', 'roles'] - (init_custom_path_actual.files | map(attribute='path') | list)[2] | basename in ['docs', 'plugins', 'roles'] +- name: add a directory to the init collection path to test that --force removes it + file: + state: directory + path: "{{ galaxy_dir }}/scratch/custom-init-dir/ansible_test2/my_collection/remove_me" + +- name: create collection with custom init path + command: ansible-galaxy collection init ansible_test2.my_collection --init-path "{{ galaxy_dir }}/scratch/custom-init-dir" --force {{ galaxy_verbosity }} + register: init_custom_path + +- name: get result of create default skeleton + find: + path: '{{ galaxy_dir }}/scratch/custom-init-dir/ansible_test2/my_collection' + file_type: directory + register: init_custom_path_actual + +- name: assert create collection with custom init path + assert: + that: + - '"Collection ansible_test2.my_collection was created successfully" in init_custom_path.stdout' + - init_custom_path_actual.files | length == 3 + - (init_custom_path_actual.files | map(attribute='path') | list)[0] | basename in ['docs', 'plugins', 'roles'] + - (init_custom_path_actual.files | map(attribute='path') | list)[1] | basename in ['docs', 'plugins', 'roles'] + - (init_custom_path_actual.files | map(attribute='path') | list)[2] | basename in ['docs', 'plugins', 'roles'] + +- name: create collection in cwd with custom init path + command: ansible-galaxy collection init ansible_test2.my_collection --init-path ../../ --force {{ galaxy_verbosity }} + args: + chdir: "{{ galaxy_dir }}/scratch/custom-init-dir/ansible_test2/my_collection" + register: init_custom_path + +- name: get result of create default skeleton + find: + path: '{{ galaxy_dir }}/scratch/custom-init-dir/ansible_test2/my_collection' + file_type: directory + register: init_custom_path_actual + +- name: assert create collection with custom init path + assert: + that: + - '"Collection ansible_test2.my_collection was created successfully" in init_custom_path.stdout' + - init_custom_path_actual.files | length == 3 + - (init_custom_path_actual.files | map(attribute='path') | list)[0] | basename in ['docs', 'plugins', 'roles'] + - (init_custom_path_actual.files | map(attribute='path') | list)[1] | basename in ['docs', 'plugins', 'roles'] + - (init_custom_path_actual.files | map(attribute='path') | list)[2] | basename in ['docs', 'plugins', 'roles'] + - name: create collection for ignored files and folders command: ansible-galaxy collection init ansible_test.ignore args: diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/install.yml b/test/integration/targets/ansible-galaxy-collection/tasks/install.yml index d345031b..0068e76d 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/install.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/install.yml @@ -1,5 +1,5 @@ --- -- name: create test collection install directory - {{ test_name }} +- name: create test collection install directory - {{ test_id }} file: path: '{{ galaxy_dir }}/ansible_collections' state: directory @@ -36,24 +36,24 @@ path: '{{ galaxy_dir }}/ansible_collections/namespace1' state: absent -- name: install simple collection with implicit path - {{ test_name }} +- name: install simple collection with implicit path - {{ test_id }} command: ansible-galaxy collection install namespace1.name1 -s '{{ test_name }}' {{ galaxy_verbosity }} environment: ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' register: install_normal -- name: get installed files of install simple collection with implicit path - {{ test_name }} +- name: get installed files of install simple collection with implicit path - {{ test_id }} find: path: '{{ galaxy_dir }}/ansible_collections/namespace1/name1' file_type: file register: install_normal_files -- name: get the manifest of install simple collection with implicit path - {{ test_name }} +- name: get the manifest of install simple collection with implicit path - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/namespace1/name1/MANIFEST.json' register: install_normal_manifest -- name: assert install simple collection with implicit path - {{ test_name }} +- name: assert install simple collection with implicit path - {{ test_id }} assert: that: - '"Installing ''namespace1.name1:1.0.9'' to" in install_normal.stdout' @@ -63,43 +63,43 @@ - install_normal_files.files[2].path | basename in ['MANIFEST.json', 'FILES.json', 'README.md'] - (install_normal_manifest.content | b64decode | from_json).collection_info.version == '1.0.9' -- name: install existing without --force - {{ test_name }} +- name: install existing without --force - {{ test_id }} command: ansible-galaxy collection install namespace1.name1 -s '{{ test_name }}' {{ galaxy_verbosity }} environment: ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' register: install_existing_no_force -- name: assert install existing without --force - {{ test_name }} +- name: assert install existing without --force - {{ test_id }} assert: that: - '"Nothing to do. All requested collections are already installed" in install_existing_no_force.stdout' -- name: install existing with --force - {{ test_name }} +- name: install existing with --force - {{ test_id }} command: ansible-galaxy collection install namespace1.name1 -s '{{ test_name }}' --force {{ galaxy_verbosity }} environment: ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' register: install_existing_force -- name: assert install existing with --force - {{ test_name }} +- name: assert install existing with --force - {{ test_id }} assert: that: - '"Installing ''namespace1.name1:1.0.9'' to" in install_existing_force.stdout' -- name: remove test installed collection - {{ test_name }} +- name: remove test installed collection - {{ test_id }} file: path: '{{ galaxy_dir }}/ansible_collections/namespace1' state: absent -- name: install pre-release as explicit version to custom dir - {{ test_name }} +- name: install pre-release as explicit version to custom dir - {{ test_id }} command: ansible-galaxy collection install 'namespace1.name1:1.1.0-beta.1' -s '{{ test_name }}' -p '{{ galaxy_dir }}/ansible_collections' {{ galaxy_verbosity }} register: install_prerelease -- name: get result of install pre-release as explicit version to custom dir - {{ test_name }} +- name: get result of install pre-release as explicit version to custom dir - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/namespace1/name1/MANIFEST.json' register: install_prerelease_actual -- name: assert install pre-release as explicit version to custom dir - {{ test_name }} +- name: assert install pre-release as explicit version to custom dir - {{ test_id }} assert: that: - '"Installing ''namespace1.name1:1.1.0-beta.1'' to" in install_prerelease.stdout' @@ -110,22 +110,22 @@ path: '{{ galaxy_dir }}/ansible_collections/namespace1/name1' state: absent -- name: install pre-release version with --pre to custom dir - {{ test_name }} +- name: install pre-release version with --pre to custom dir - {{ test_id }} command: ansible-galaxy collection install --pre 'namespace1.name1' -s '{{ test_name }}' -p '{{ galaxy_dir }}/ansible_collections' {{ galaxy_verbosity }} register: install_prerelease -- name: get result of install pre-release version with --pre to custom dir - {{ test_name }} +- name: get result of install pre-release version with --pre to custom dir - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/namespace1/name1/MANIFEST.json' register: install_prerelease_actual -- name: assert install pre-release version with --pre to custom dir - {{ test_name }} +- name: assert install pre-release version with --pre to custom dir - {{ test_id }} assert: that: - '"Installing ''namespace1.name1:1.1.0-beta.1'' to" in install_prerelease.stdout' - (install_prerelease_actual.content | b64decode | from_json).collection_info.version == '1.1.0-beta.1' -- name: install multiple collections with dependencies - {{ test_name }} +- name: install multiple collections with dependencies - {{ test_id }} command: ansible-galaxy collection install parent_dep.parent_collection:1.0.0 namespace2.name -s {{ test_name }} {{ galaxy_verbosity }} args: chdir: '{{ galaxy_dir }}/ansible_collections' @@ -134,7 +134,7 @@ ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg' register: install_multiple_with_dep -- name: get result of install multiple collections with dependencies - {{ test_name }} +- name: get result of install multiple collections with dependencies - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/{{ collection.namespace }}/{{ collection.name }}/MANIFEST.json' register: install_multiple_with_dep_actual @@ -150,7 +150,7 @@ - namespace: child_dep name: child_dep2 -- name: assert install multiple collections with dependencies - {{ test_name }} +- name: assert install multiple collections with dependencies - {{ test_id }} assert: that: - (install_multiple_with_dep_actual.results[0].content | b64decode | from_json).collection_info.version == '1.0.0' @@ -158,7 +158,7 @@ - (install_multiple_with_dep_actual.results[2].content | b64decode | from_json).collection_info.version == '0.9.9' - (install_multiple_with_dep_actual.results[3].content | b64decode | from_json).collection_info.version == '1.2.2' -- name: expect failure with dep resolution failure +- name: expect failure with dep resolution failure - {{ test_id }} command: ansible-galaxy collection install fail_namespace.fail_collection -s {{ test_name }} {{ galaxy_verbosity }} register: fail_dep_mismatch failed_when: @@ -173,23 +173,23 @@ force_basic_auth: true register: artifact_url_response -- name: download a collection for an offline install - {{ test_name }} +- name: download a collection for an offline install - {{ test_id }} get_url: url: '{{ artifact_url_response.json.download_url }}' dest: '{{ galaxy_dir }}/namespace3.tar.gz' -- name: install a collection from a tarball - {{ test_name }} +- name: install a collection from a tarball - {{ test_id }} command: ansible-galaxy collection install '{{ galaxy_dir }}/namespace3.tar.gz' {{ galaxy_verbosity }} register: install_tarball environment: ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' -- name: get result of install collection from a tarball - {{ test_name }} +- name: get result of install collection from a tarball - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/namespace3/name/MANIFEST.json' register: install_tarball_actual -- name: assert install a collection from a tarball - {{ test_name }} +- name: assert install a collection from a tarball - {{ test_id }} assert: that: - '"Installing ''namespace3.name:1.0.0'' to" in install_tarball.stdout' @@ -270,22 +270,22 @@ - "{{ galaxy_dir }}/scratch/tmp_parent/" - "{{ galaxy_dir }}/tmp_parent-name-1.0.0.tar.gz" -- name: setup bad tarball - {{ test_name }} +- name: setup bad tarball - {{ test_id }} script: build_bad_tar.py {{ galaxy_dir | quote }} -- name: fail to install a collection from a bad tarball - {{ test_name }} +- name: fail to install a collection from a bad tarball - {{ test_id }} command: ansible-galaxy collection install '{{ galaxy_dir }}/suspicious-test-1.0.0.tar.gz' {{ galaxy_verbosity }} register: fail_bad_tar failed_when: fail_bad_tar.rc != 1 and "Cannot extract tar entry '../../outside.sh' as it will be placed outside the collection directory" not in fail_bad_tar.stderr environment: ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' -- name: get result of failed collection install - {{ test_name }} +- name: get result of failed collection install - {{ test_id }} stat: path: '{{ galaxy_dir }}/ansible_collections\suspicious' register: fail_bad_tar_actual -- name: assert result of failed collection install - {{ test_name }} +- name: assert result of failed collection install - {{ test_id }} assert: that: - not fail_bad_tar_actual.stat.exists @@ -298,24 +298,24 @@ force_basic_auth: true register: artifact_url_response -- name: install a collection from a URI - {{ test_name }} +- name: install a collection from a URI - {{ test_id }} command: ansible-galaxy collection install {{ artifact_url_response.json.download_url}} {{ galaxy_verbosity }} register: install_uri environment: ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' -- name: get result of install collection from a URI - {{ test_name }} +- name: get result of install collection from a URI - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/namespace4/name/MANIFEST.json' register: install_uri_actual -- name: assert install a collection from a URI - {{ test_name }} +- name: assert install a collection from a URI - {{ test_id }} assert: that: - '"Installing ''namespace4.name:1.0.0'' to" in install_uri.stdout' - (install_uri_actual.content | b64decode | from_json).collection_info.version == '1.0.0' -- name: fail to install a collection with an undefined URL - {{ test_name }} +- name: fail to install a collection with an undefined URL - {{ test_id }} command: ansible-galaxy collection install namespace5.name {{ galaxy_verbosity }} register: fail_undefined_server failed_when: '"No setting was provided for required configuration plugin_type: galaxy_server plugin: undefined" not in fail_undefined_server.stderr' @@ -324,25 +324,25 @@ - when: not requires_auth block: - - name: install a collection with an empty server list - {{ test_name }} + - name: install a collection with an empty server list - {{ test_id }} command: ansible-galaxy collection install namespace5.name -s '{{ test_server }}' {{ galaxy_verbosity }} register: install_empty_server_list environment: ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' ANSIBLE_GALAXY_SERVER_LIST: '' - - name: get result of a collection with an empty server list - {{ test_name }} + - name: get result of a collection with an empty server list - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/namespace5/name/MANIFEST.json' register: install_empty_server_list_actual - - name: assert install a collection with an empty server list - {{ test_name }} + - name: assert install a collection with an empty server list - {{ test_id }} assert: that: - '"Installing ''namespace5.name:1.0.0'' to" in install_empty_server_list.stdout' - (install_empty_server_list_actual.content | b64decode | from_json).collection_info.version == '1.0.0' -- name: create test requirements file with both roles and collections - {{ test_name }} +- name: create test requirements file with both roles and collections - {{ test_id }} copy: content: | collections: @@ -366,13 +366,13 @@ - "'unrecognized arguments: --keyring' in invalid_opt.stderr" # Need to run with -vvv to validate the roles will be skipped msg -- name: install collections only with requirements-with-role.yml - {{ test_name }} +- name: install collections only with requirements-with-role.yml - {{ test_id }} command: ansible-galaxy collection install -r '{{ galaxy_dir }}/ansible_collections/requirements-with-role.yml' -s '{{ test_name }}' -vvv register: install_req_collection environment: ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' -- name: get result of install collections only with requirements-with-roles.yml - {{ test_name }} +- name: get result of install collections only with requirements-with-roles.yml - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/{{ collection }}/name/MANIFEST.json' register: install_req_collection_actual @@ -382,7 +382,7 @@ - namespace6 - namespace7 -- name: assert install collections only with requirements-with-role.yml - {{ test_name }} +- name: assert install collections only with requirements-with-role.yml - {{ test_id }} assert: that: - '"contains roles which will be ignored" in install_req_collection.stdout' @@ -391,7 +391,7 @@ - (install_req_collection_actual.results[0].content | b64decode | from_json).collection_info.version == '1.0.0' - (install_req_collection_actual.results[1].content | b64decode | from_json).collection_info.version == '1.0.0' -- name: create test requirements file with just collections - {{ test_name }} +- name: create test requirements file with just collections - {{ test_id }} copy: content: | collections: @@ -399,13 +399,13 @@ - name: namespace9.name dest: '{{ galaxy_dir }}/ansible_collections/requirements.yaml' -- name: install collections with ansible-galaxy install - {{ test_name }} +- name: install collections with ansible-galaxy install - {{ test_id }} command: ansible-galaxy install -r '{{ galaxy_dir }}/ansible_collections/requirements.yaml' -s '{{ test_name }}' register: install_req environment: ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' -- name: get result of install collections with ansible-galaxy install - {{ test_name }} +- name: get result of install collections with ansible-galaxy install - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/{{ collection }}/name/MANIFEST.json' register: install_req_actual @@ -415,7 +415,7 @@ - namespace8 - namespace9 -- name: assert install collections with ansible-galaxy install - {{ test_name }} +- name: assert install collections with ansible-galaxy install - {{ test_id }} assert: that: - '"Installing ''namespace8.name:1.0.0'' to" in install_req.stdout' @@ -485,7 +485,7 @@ - required_together is failed - '"ERROR! Signatures were provided to verify namespace1.name1 but no keyring was configured." in required_together.stderr' -- name: install collections with ansible-galaxy install -r with invalid signatures - {{ test_name }} +- name: install collections with ansible-galaxy install -r with invalid signatures - {{ test_id }} # Note that --keyring is a valid option for 'ansible-galaxy install -r ...', not just 'ansible-galaxy collection ...' command: ansible-galaxy install -r {{ req_file }} -s {{ test_name }} --keyring {{ keyring }} {{ galaxy_verbosity }} register: install_req @@ -497,7 +497,7 @@ ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' ANSIBLE_GALAXY_REQUIRED_VALID_SIGNATURE_COUNT: all -- name: assert invalid signature is fatal with ansible-galaxy install - {{ test_name }} +- name: assert invalid signature is fatal with ansible-galaxy install - {{ test_id }} assert: that: - install_req is failed @@ -509,7 +509,7 @@ - '"Installing ''namespace9.name:1.0.0'' to" not in install_req.stdout' # This command is hardcoded with -vvvv purposefully to evaluate extra verbosity messages -- name: install collections with ansible-galaxy install and --ignore-errors - {{ test_name }} +- name: install collections with ansible-galaxy install and --ignore-errors - {{ test_id }} command: ansible-galaxy install -r {{ req_file }} {{ cli_opts }} -vvvv register: install_req vars: @@ -520,7 +520,7 @@ ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' ANSIBLE_GALAXY_REQUIRED_VALID_SIGNATURE_COUNT: all -- name: get result of install collections with ansible-galaxy install - {{ test_name }} +- name: get result of install collections with ansible-galaxy install - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/{{ collection }}/name/MANIFEST.json' register: install_req_actual @@ -531,7 +531,7 @@ - namespace9 # SIVEL -- name: assert invalid signature is not fatal with ansible-galaxy install --ignore-errors - {{ test_name }} +- name: assert invalid signature is not fatal with ansible-galaxy install --ignore-errors - {{ test_id }} assert: that: - install_req is success @@ -558,7 +558,7 @@ - namespace8 - namespace9 -- name: install collections with only one valid signature using ansible-galaxy install - {{ test_name }} +- name: install collections with only one valid signature using ansible-galaxy install - {{ test_id }} command: ansible-galaxy install -r {{ req_file }} {{ cli_opts }} {{ galaxy_verbosity }} register: install_req vars: @@ -568,7 +568,7 @@ environment: ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' -- name: get result of install collections with ansible-galaxy install - {{ test_name }} +- name: get result of install collections with ansible-galaxy install - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/{{ collection }}/name/MANIFEST.json' register: install_req_actual @@ -579,7 +579,7 @@ - namespace8 - namespace9 -- name: assert just one valid signature is not fatal with ansible-galaxy install - {{ test_name }} +- name: assert just one valid signature is not fatal with ansible-galaxy install - {{ test_id }} assert: that: - install_req is success @@ -619,7 +619,7 @@ ANSIBLE_GALAXY_REQUIRED_VALID_SIGNATURE_COUNT: all ANSIBLE_GALAXY_IGNORE_SIGNATURE_STATUS_CODES: BADSIG # cli option is appended and both status codes are ignored -- name: get result of install collections with ansible-galaxy install - {{ test_name }} +- name: get result of install collections with ansible-galaxy install - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/{{ collection }}/name/MANIFEST.json' register: install_req_actual @@ -630,7 +630,7 @@ - namespace8 - namespace9 -- name: assert invalid signature is not fatal with ansible-galaxy install - {{ test_name }} +- name: assert invalid signature is not fatal with ansible-galaxy install - {{ test_id }} assert: that: - install_req is success @@ -675,24 +675,24 @@ # name: cache # version: 1.0.{{ cache_version_build }} # -#- name: make sure the cache version list is ignored on a collection version change - {{ test_name }} +#- name: make sure the cache version list is ignored on a collection version change - {{ test_id }} # command: ansible-galaxy collection install cache.cache -s '{{ test_name }}' --force -vvv # register: install_cached_update # environment: # ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' # -#- name: get result of cache version list is ignored on a collection version change - {{ test_name }} +#- name: get result of cache version list is ignored on a collection version change - {{ test_id }} # slurp: # path: '{{ galaxy_dir }}/ansible_collections/cache/cache/MANIFEST.json' # register: install_cached_update_actual # -#- name: assert cache version list is ignored on a collection version change - {{ test_name }} +#- name: assert cache version list is ignored on a collection version change - {{ test_id }} # assert: # that: # - '"Installing ''cache.cache:1.0.{{ cache_version_build }}'' to" in install_cached_update.stdout' # - (install_cached_update_actual.content | b64decode | from_json).collection_info.version == '1.0.' ~ cache_version_build -- name: install collection with symlink - {{ test_name }} +- name: install collection with symlink - {{ test_id }} command: ansible-galaxy collection install symlink.symlink -s '{{ test_name }}' {{ galaxy_verbosity }} environment: ANSIBLE_COLLECTIONS_PATHS: '{{ galaxy_dir }}/ansible_collections' @@ -703,7 +703,7 @@ recurse: yes file_type: any -- name: get result of install collection with symlink - {{ test_name }} +- name: get result of install collection with symlink - {{ test_id }} stat: path: '{{ galaxy_dir }}/ansible_collections/symlink/symlink/{{ path }}' register: install_symlink_actual @@ -717,7 +717,7 @@ - docs-link - docs-link/REÅDMÈ.md -- name: assert install collection with symlink - {{ test_name }} +- name: assert install collection with symlink - {{ test_id }} assert: that: - '"Installing ''symlink.symlink:1.0.0'' to" in install_symlink.stdout' @@ -733,18 +733,18 @@ - install_symlink_actual.results[5].stat.islnk - install_symlink_actual.results[5].stat.lnk_target == '../REÅDMÈ.md' -- name: remove install directory for the next test because parent_dep.parent_collection was installed - {{ test_name }} +- name: remove install directory for the next test because parent_dep.parent_collection was installed - {{ test_id }} file: path: '{{ galaxy_dir }}/ansible_collections' state: absent -- name: install collection and dep compatible with multiple requirements - {{ test_name }} +- name: install collection and dep compatible with multiple requirements - {{ test_id }} command: ansible-galaxy collection install parent_dep.parent_collection parent_dep2.parent_collection environment: ANSIBLE_COLLECTIONS_PATHS: '{{ galaxy_dir }}/ansible_collections' register: install_req -- name: assert install collections with ansible-galaxy install - {{ test_name }} +- name: assert install collections with ansible-galaxy install - {{ test_id }} assert: that: - '"Installing ''parent_dep.parent_collection:1.0.0'' to" in install_req.stdout' @@ -760,18 +760,18 @@ state: directory path: '{{ galaxy_dir }}/ansible_collections/unrelated_namespace/collection_without_metadata/plugins' - - name: install a collection to the same installation directory - {{ test_name }} + - name: install a collection to the same installation directory - {{ test_id }} command: ansible-galaxy collection install namespace1.name1 environment: ANSIBLE_COLLECTIONS_PATHS: '{{ galaxy_dir }}/ansible_collections' register: install_req - - name: assert installed collections with ansible-galaxy install - {{ test_name }} + - name: assert installed collections with ansible-galaxy install - {{ test_id }} assert: that: - '"Installing ''namespace1.name1:1.0.9'' to" in install_req.stdout' -- name: remove test collection install directory - {{ test_name }} +- name: remove test collection install directory - {{ test_id }} file: path: '{{ galaxy_dir }}/ansible_collections' state: absent @@ -964,10 +964,10 @@ path: '{{ galaxy_dir }}/ansible_collections/namespace1' state: absent -- name: download collections with pre-release dep - {{ test_name }} +- name: download collections with pre-release dep - {{ test_id }} command: ansible-galaxy collection download dep_with_beta.parent namespace1.name1:1.1.0-beta.1 -p '{{ galaxy_dir }}/scratch' -- name: install collection with concrete pre-release dep - {{ test_name }} +- name: install collection with concrete pre-release dep - {{ test_id }} command: ansible-galaxy collection install -r '{{ galaxy_dir }}/scratch/requirements.yml' args: chdir: '{{ galaxy_dir }}/scratch' @@ -975,7 +975,7 @@ ANSIBLE_COLLECTIONS_PATHS: '{{ galaxy_dir }}/ansible_collections' register: install_concrete_pre -- name: get result of install collections with concrete pre-release dep - {{ test_name }} +- name: get result of install collections with concrete pre-release dep - {{ test_id }} slurp: path: '{{ galaxy_dir }}/ansible_collections/{{ collection }}/MANIFEST.json' register: install_concrete_pre_actual @@ -985,7 +985,7 @@ - namespace1/name1 - dep_with_beta/parent -- name: assert install collections with ansible-galaxy install - {{ test_name }} +- name: assert install collections with ansible-galaxy install - {{ test_id }} assert: that: - '"Installing ''namespace1.name1:1.1.0-beta.1'' to" in install_concrete_pre.stdout' @@ -993,7 +993,7 @@ - (install_concrete_pre_actual.results[0].content | b64decode | from_json).collection_info.version == '1.1.0-beta.1' - (install_concrete_pre_actual.results[1].content | b64decode | from_json).collection_info.version == '1.0.0' -- name: remove collection dir after round of testing - {{ test_name }} +- name: remove collection dir after round of testing - {{ test_id }} file: path: '{{ galaxy_dir }}/ansible_collections' state: absent diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/list.yml b/test/integration/targets/ansible-galaxy-collection/tasks/list.yml index 331e0a1c..b8d63492 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/list.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/list.yml @@ -4,6 +4,9 @@ - 'dev.collection1' - 'dev.collection2' - 'dev.collection3' + - 'dev.collection4' + - 'dev.collection5' + - 'dev.collection6' - name: replace the default version of the collections lineinfile: @@ -18,6 +21,30 @@ - name: "collection3" version: "version: ''" +- name: set the namespace, name, and version keys to None + lineinfile: + path: "{{ galaxy_dir }}/dev/ansible_collections/dev/collection4/galaxy.yml" + line: "{{ item.after }}" + regexp: "{{ item.before }}" + loop: + - before: "^namespace: dev" + after: "namespace:" + - before: "^name: collection4" + after: "name:" + - before: "^version: 1.0.0" + after: "version:" + +- name: replace galaxy.yml content with a string + copy: + content: "invalid" + dest: "{{ galaxy_dir }}/dev/ansible_collections/dev/collection5/galaxy.yml" + +- name: remove galaxy.yml key required by build + lineinfile: + path: "{{ galaxy_dir }}/dev/ansible_collections/dev/collection6/galaxy.yml" + line: "version: 1.0.0" + state: absent + - name: list collections in development without semver versions command: ansible-galaxy collection list {{ galaxy_verbosity }} register: list_result @@ -30,6 +57,9 @@ # Note the version displayed is the 'placeholder' string rather than "*" since it is not falsey - "'dev.collection2 placeholder' in list_result.stdout" - "'dev.collection3 *' in list_result.stdout" + - "'dev.collection4 *' in list_result.stdout" + - "'dev.collection5 *' in list_result.stdout" + - "'dev.collection6 *' in list_result.stdout" - name: list collections in human format command: ansible-galaxy collection list --format human @@ -43,6 +73,8 @@ # Note the version displayed is the 'placeholder' string rather than "*" since it is not falsey - "'dev.collection2 placeholder' in list_result_human.stdout" - "'dev.collection3 *' in list_result_human.stdout" + - "'dev.collection5 *' in list_result.stdout" + - "'dev.collection6 *' in list_result.stdout" - name: list collections in yaml format command: ansible-galaxy collection list --format yaml @@ -52,10 +84,12 @@ - assert: that: - - "item.value | length == 3" + - "item.value | length == 6" - "item.value['dev.collection1'].version == '*'" - "item.value['dev.collection2'].version == 'placeholder'" - "item.value['dev.collection3'].version == '*'" + - "item.value['dev.collection5'].version == '*'" + - "item.value['dev.collection6'].version == '*'" with_dict: "{{ list_result_yaml.stdout | from_yaml }}" - name: list collections in json format @@ -66,10 +100,12 @@ - assert: that: - - "item.value | length == 3" + - "item.value | length == 6" - "item.value['dev.collection1'].version == '*'" - "item.value['dev.collection2'].version == 'placeholder'" - "item.value['dev.collection3'].version == '*'" + - "item.value['dev.collection5'].version == '*'" + - "item.value['dev.collection6'].version == '*'" with_dict: "{{ list_result_json.stdout | from_json }}" - name: list single collection in json format diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/main.yml b/test/integration/targets/ansible-galaxy-collection/tasks/main.yml index 598784d3..063b7f08 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/main.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/main.yml @@ -50,6 +50,12 @@ src: ansible.cfg.j2 dest: '{{ galaxy_dir }}/ansible.cfg' +- name: test install command using an unsupported version of resolvelib + include_tasks: unsupported_resolvelib.yml + loop: "{{ unsupported_resolvelib_versions }}" + loop_control: + loop_var: resolvelib_version + - name: run ansible-galaxy collection publish tests for {{ test_name }} include_tasks: publish.yml args: @@ -92,6 +98,7 @@ - name: run ansible-galaxy collection install tests for {{ test_name }} include_tasks: install.yml vars: + test_id: '{{ item.name }}' test_name: '{{ item.name }}' test_server: '{{ item.server }}' vX: '{{ "v3/" if item.v3|default(false) else "v2/" }}' @@ -111,6 +118,16 @@ server: '{{ pulp_server }}published/api/' v3: true +- name: test installing and downloading collections with the range of supported resolvelib versions + include_tasks: supported_resolvelib.yml + args: + apply: + environment: + ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg' + loop: '{{ supported_resolvelib_versions }}' + loop_control: + loop_var: resolvelib_version + - name: publish collection with a dep on another server setup_collections: server: secondary diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/revoke_gpg_key.yml b/test/integration/targets/ansible-galaxy-collection/tasks/revoke_gpg_key.yml index a766d8ea..7a49eeed 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/revoke_gpg_key.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/revoke_gpg_key.yml @@ -1,6 +1,6 @@ - name: generate revocation certificate expect: - command: "gpg --homedir {{ gpg_homedir }} --output {{ gpg_homedir }}/revoke.asc --gen-revoke {{ fingerprint }}" + command: "gpg --homedir {{ gpg_homedir }} --pinentry-mode loopback --output {{ gpg_homedir }}/revoke.asc --gen-revoke {{ fingerprint }}" responses: "Create a revocation certificate for this key": "y" "Please select the reason for the revocation": "0" diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/setup_gpg.yml b/test/integration/targets/ansible-galaxy-collection/tasks/setup_gpg.yml index 93d532f6..ddc4d8a6 100644 --- a/test/integration/targets/ansible-galaxy-collection/tasks/setup_gpg.yml +++ b/test/integration/targets/ansible-galaxy-collection/tasks/setup_gpg.yml @@ -12,7 +12,7 @@ register: user - name: generate key for user with gpg - command: "gpg --no-tty --homedir {{ gpg_homedir }} --passphrase SECRET --pinentry-mode loopback --quick-gen-key {{ user.stdout }} default default" + command: "gpg --no-tty --homedir {{ gpg_homedir }} --passphrase '' --pinentry-mode loopback --quick-gen-key {{ user.stdout }} default default" - name: list gpg keys for user command: "gpg --no-tty --homedir {{ gpg_homedir }} --list-keys {{ user.stdout }}" diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/supported_resolvelib.yml b/test/integration/targets/ansible-galaxy-collection/tasks/supported_resolvelib.yml new file mode 100644 index 00000000..763c5a19 --- /dev/null +++ b/test/integration/targets/ansible-galaxy-collection/tasks/supported_resolvelib.yml @@ -0,0 +1,44 @@ +- vars: + venv_cmd: "{{ ansible_python_interpreter ~ ' -m venv' }}" + venv_dest: "{{ galaxy_dir }}/test_venv_{{ resolvelib_version }}" + block: + - name: install another version of resolvelib that is supported by ansible-galaxy + pip: + name: resolvelib + version: "{{ resolvelib_version }}" + state: present + virtualenv_command: "{{ venv_cmd }}" + virtualenv: "{{ venv_dest }}" + virtualenv_site_packages: True + + - include_tasks: fail_fast_resolvelib.yml + args: + apply: + environment: + PATH: "{{ venv_dest }}/bin:{{ ansible_env.PATH }}" + ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg' + + - include_tasks: install.yml + vars: + test_name: pulp_v3 + test_id: '{{ test_name }} (resolvelib {{ resolvelib_version }})' + test_server: '{{ pulp_server }}published/api/' + vX: "v3/" + requires_auth: false + args: + apply: + environment: + PATH: "{{ venv_dest }}/bin:{{ ansible_env.PATH }}" + ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg' + + - include_tasks: download.yml + args: + apply: + environment: + PATH: "{{ venv_dest }}/bin:{{ ansible_env.PATH }}" + ANSIBLE_CONFIG: '{{ galaxy_dir }}/ansible.cfg' + always: + - name: remove test venv + file: + path: "{{ venv_dest }}" + state: absent diff --git a/test/integration/targets/ansible-galaxy-collection/tasks/unsupported_resolvelib.yml b/test/integration/targets/ansible-galaxy-collection/tasks/unsupported_resolvelib.yml new file mode 100644 index 00000000..a208b295 --- /dev/null +++ b/test/integration/targets/ansible-galaxy-collection/tasks/unsupported_resolvelib.yml @@ -0,0 +1,44 @@ +- vars: + venv_cmd: "{{ ansible_python_interpreter ~ ' -m venv' }}" + venv_dest: "{{ galaxy_dir }}/test_resolvelib_{{ resolvelib_version }}" + block: + - name: install another version of resolvelib that is unsupported by ansible-galaxy + pip: + name: resolvelib + version: "{{ resolvelib_version }}" + state: present + virtualenv_command: "{{ venv_cmd }}" + virtualenv: "{{ venv_dest }}" + virtualenv_site_packages: True + + - name: create test collection install directory - {{ test_name }} + file: + path: '{{ galaxy_dir }}/ansible_collections' + state: directory + + - name: install simple collection from first accessible server (expected failure) + command: "ansible-galaxy collection install namespace1.name1 {{ galaxy_verbosity }}" + environment: + ANSIBLE_COLLECTIONS_PATH: '{{ galaxy_dir }}/ansible_collections' + PATH: "{{ venv_dest }}/bin:{{ ansible_env.PATH }}" + register: resolvelib_version_error + ignore_errors: yes + + - assert: + that: + - resolvelib_version_error is failed + - resolvelib_version_error.stderr | regex_search(error) + vars: + error: "({{ import_error }}|{{ compat_error }})" + import_error: "Failed to import resolvelib" + compat_error: "ansible-galaxy requires resolvelib<{{major_minor_patch}},>={{major_minor_patch}}" + major_minor_patch: "[0-9]\\d*\\.[0-9]\\d*\\.[0-9]\\d*" + + always: + - name: cleanup venv and install directory + file: + path: '{{ galaxy_dir }}/ansible_collections' + state: absent + loop: + - '{{ galaxy_dir }}/ansible_collections' + - '{{ venv_dest }}' diff --git a/test/integration/targets/ansible-galaxy-collection/vars/main.yml b/test/integration/targets/ansible-galaxy-collection/vars/main.yml index 604ff1ab..12e968aa 100644 --- a/test/integration/targets/ansible-galaxy-collection/vars/main.yml +++ b/test/integration/targets/ansible-galaxy-collection/vars/main.yml @@ -2,6 +2,16 @@ galaxy_verbosity: "{{ '' if not ansible_verbosity else '-' ~ ('v' * ansible_verb gpg_homedir: "{{ galaxy_dir }}/gpg" +supported_resolvelib_versions: + - "0.5.3" # Oldest supported + - "0.6.0" + - "0.7.0" + - "0.8.0" + +unsupported_resolvelib_versions: + - "0.2.0" # Fails on import + - "0.5.1" + pulp_repositories: - published - secondary diff --git a/test/integration/targets/ansible-test-config-invalid/aliases b/test/integration/targets/ansible-test-config-invalid/aliases new file mode 100644 index 00000000..193276cc --- /dev/null +++ b/test/integration/targets/ansible-test-config-invalid/aliases @@ -0,0 +1,4 @@ +shippable/posix/group1 # runs in the distro test containers +shippable/generic/group1 # runs in the default test container +context/controller +needs/target/collection diff --git a/test/integration/targets/ansible-test-config-invalid/ansible_collections/ns/col/tests/config.yml b/test/integration/targets/ansible-test-config-invalid/ansible_collections/ns/col/tests/config.yml new file mode 100644 index 00000000..9977a283 --- /dev/null +++ b/test/integration/targets/ansible-test-config-invalid/ansible_collections/ns/col/tests/config.yml @@ -0,0 +1 @@ +invalid diff --git a/test/integration/targets/ansible-test-config-invalid/ansible_collections/ns/col/tests/integration/targets/test/aliases b/test/integration/targets/ansible-test-config-invalid/ansible_collections/ns/col/tests/integration/targets/test/aliases new file mode 100644 index 00000000..1af1cf90 --- /dev/null +++ b/test/integration/targets/ansible-test-config-invalid/ansible_collections/ns/col/tests/integration/targets/test/aliases @@ -0,0 +1 @@ +context/controller diff --git a/test/integration/targets/ansible-test-config-invalid/ansible_collections/ns/col/tests/integration/targets/test/runme.sh b/test/integration/targets/ansible-test-config-invalid/ansible_collections/ns/col/tests/integration/targets/test/runme.sh new file mode 100755 index 00000000..f1f641af --- /dev/null +++ b/test/integration/targets/ansible-test-config-invalid/ansible_collections/ns/col/tests/integration/targets/test/runme.sh @@ -0,0 +1 @@ +#!/usr/bin/env bash diff --git a/test/integration/targets/ansible-test-config-invalid/ansible_collections/ns/col/tests/unit/plugins/module_utils/test_test.py b/test/integration/targets/ansible-test-config-invalid/ansible_collections/ns/col/tests/unit/plugins/module_utils/test_test.py new file mode 100644 index 00000000..06e7782e --- /dev/null +++ b/test/integration/targets/ansible-test-config-invalid/ansible_collections/ns/col/tests/unit/plugins/module_utils/test_test.py @@ -0,0 +1,2 @@ +def test_me(): + pass diff --git a/test/integration/targets/ansible-test-config-invalid/runme.sh b/test/integration/targets/ansible-test-config-invalid/runme.sh new file mode 100755 index 00000000..6ff2d406 --- /dev/null +++ b/test/integration/targets/ansible-test-config-invalid/runme.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Make sure that ansible-test continues to work when content config is invalid. + +set -eu + +source ../collection/setup.sh + +set -x + +ansible-test sanity --test import --python "${ANSIBLE_TEST_PYTHON_VERSION}" --color --venv -v +ansible-test units --python "${ANSIBLE_TEST_PYTHON_VERSION}" --color --venv -v +ansible-test integration --color --venv -v diff --git a/test/integration/targets/ansible-test-config/aliases b/test/integration/targets/ansible-test-config/aliases new file mode 100644 index 00000000..193276cc --- /dev/null +++ b/test/integration/targets/ansible-test-config/aliases @@ -0,0 +1,4 @@ +shippable/posix/group1 # runs in the distro test containers +shippable/generic/group1 # runs in the default test container +context/controller +needs/target/collection diff --git a/test/integration/targets/ansible-test-config/ansible_collections/ns/col/plugins/module_utils/test.py b/test/integration/targets/ansible-test-config/ansible_collections/ns/col/plugins/module_utils/test.py new file mode 100644 index 00000000..962dba2b --- /dev/null +++ b/test/integration/targets/ansible-test-config/ansible_collections/ns/col/plugins/module_utils/test.py @@ -0,0 +1,14 @@ +import sys +import os + + +def version_to_str(value): + return '.'.join(str(v) for v in value) + + +controller_min_python_version = tuple(int(v) for v in os.environ['ANSIBLE_CONTROLLER_MIN_PYTHON_VERSION'].split('.')) +current_python_version = sys.version_info[:2] + +if current_python_version < controller_min_python_version: + raise Exception('Current Python version %s is lower than the minimum controller Python version of %s. ' + 'Did the collection config get ignored?' % (version_to_str(current_python_version), version_to_str(controller_min_python_version))) diff --git a/test/integration/targets/ansible-test-config/ansible_collections/ns/col/tests/config.yml b/test/integration/targets/ansible-test-config/ansible_collections/ns/col/tests/config.yml new file mode 100644 index 00000000..7772d7d2 --- /dev/null +++ b/test/integration/targets/ansible-test-config/ansible_collections/ns/col/tests/config.yml @@ -0,0 +1,2 @@ +modules: + python_requires: controller # allow tests to pass when run against a Python version not supported by the controller diff --git a/test/integration/targets/ansible-test-config/ansible_collections/ns/col/tests/unit/plugins/module_utils/test_test.py b/test/integration/targets/ansible-test-config/ansible_collections/ns/col/tests/unit/plugins/module_utils/test_test.py new file mode 100644 index 00000000..b320a15a --- /dev/null +++ b/test/integration/targets/ansible-test-config/ansible_collections/ns/col/tests/unit/plugins/module_utils/test_test.py @@ -0,0 +1,5 @@ +from ansible_collections.ns.col.plugins.module_utils import test + + +def test_me(): + assert test diff --git a/test/integration/targets/ansible-test-config/runme.sh b/test/integration/targets/ansible-test-config/runme.sh new file mode 100755 index 00000000..9636d04d --- /dev/null +++ b/test/integration/targets/ansible-test-config/runme.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Make sure that ansible-test is able to parse collection config when using a venv. + +set -eu + +source ../collection/setup.sh + +set -x + +# On systems with a Python version below the minimum controller Python version, such as the default container, this test +# will verify that the content config is working properly after delegation. Otherwise it will only verify that no errors +# occur while trying to access content config (such as missing requirements). + +ansible-test sanity --test import --color --venv -v +ansible-test units --color --venv -v diff --git a/test/integration/targets/ansible-test-docker/aliases b/test/integration/targets/ansible-test-docker/aliases index a862ab8b..c389df53 100644 --- a/test/integration/targets/ansible-test-docker/aliases +++ b/test/integration/targets/ansible-test-docker/aliases @@ -1,2 +1,3 @@ shippable/generic/group1 # Runs in the default test container so access to tools like pwsh context/controller +needs/target/collection diff --git a/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/tests/integration/targets/minimal/aliases b/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/tests/integration/targets/minimal/aliases new file mode 100644 index 00000000..1af1cf90 --- /dev/null +++ b/test/integration/targets/ansible-test-docker/ansible_collections/ns/col/tests/integration/targets/minimal/aliases @@ -0,0 +1 @@ +context/controller diff --git a/test/integration/targets/ansible-test-docker/collection-tests/docker.sh b/test/integration/targets/ansible-test-docker/collection-tests/docker.sh deleted file mode 100755 index 69372245..00000000 --- a/test/integration/targets/ansible-test-docker/collection-tests/docker.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash - -set -eux -o pipefail - -cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}" -cd "${WORK_DIR}/ansible_collections/ns/col" - -# common args for all tests -# because we are running in shippable/generic/ we are already in the default docker container -common=(--python "${ANSIBLE_TEST_PYTHON_VERSION}" --venv --venv-system-site-packages --color --truncate 0 "${@}") - -# prime the venv to work around issue with PyYAML detection in ansible-test -ansible-test sanity "${common[@]}" --test ignores - -# tests -ansible-test sanity "${common[@]}" -ansible-test units "${common[@]}" -ansible-test integration "${common[@]}" diff --git a/test/integration/targets/ansible-test-docker/runme.sh b/test/integration/targets/ansible-test-docker/runme.sh index 7c956b4f..014d3632 100755 --- a/test/integration/targets/ansible-test-docker/runme.sh +++ b/test/integration/targets/ansible-test-docker/runme.sh @@ -1,24 +1,14 @@ #!/usr/bin/env bash -set -eu -o pipefail +source ../collection/setup.sh -# tests must be executed outside of the ansible source tree -# otherwise ansible-test will test the ansible source instead of the test collection -# the temporary directory provided by ansible-test resides within the ansible source tree -tmp_dir=$(mktemp -d) +set -x -trap 'rm -rf "${tmp_dir}"' EXIT +# common args for all tests +# because we are running in shippable/generic/ we are already in the default docker container +common=(--python "${ANSIBLE_TEST_PYTHON_VERSION}" --venv --venv-system-site-packages --color --truncate 0 "${@}") -export TEST_DIR -export WORK_DIR - -TEST_DIR="$PWD" - -for test in collection-tests/*.sh; do - WORK_DIR="${tmp_dir}/$(basename "${test}" ".sh")" - mkdir "${WORK_DIR}" - echo "**********************************************************************" - echo "TEST: ${test}: STARTING" - "${test}" "${@}" || (echo "TEST: ${test}: FAILED" && exit 1) - echo "TEST: ${test}: PASSED" -done +# tests +ansible-test sanity "${common[@]}" +ansible-test units "${common[@]}" +ansible-test integration "${common[@]}" diff --git a/test/integration/targets/ansible-test-no-tty/aliases b/test/integration/targets/ansible-test-no-tty/aliases new file mode 100644 index 00000000..620c2144 --- /dev/null +++ b/test/integration/targets/ansible-test-no-tty/aliases @@ -0,0 +1,4 @@ +context/controller +shippable/posix/group1 # runs in the distro test containers +shippable/generic/group1 # runs in the default test container +needs/target/collection diff --git a/test/integration/targets/ansible-test-no-tty/ansible_collections/ns/col/run-with-pty.py b/test/integration/targets/ansible-test-no-tty/ansible_collections/ns/col/run-with-pty.py new file mode 100755 index 00000000..46391528 --- /dev/null +++ b/test/integration/targets/ansible-test-no-tty/ansible_collections/ns/col/run-with-pty.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python +"""Run a command using a PTY.""" + +import sys + +if sys.version_info < (3, 10): + import vendored_pty as pty +else: + import pty + +sys.exit(1 if pty.spawn(sys.argv[1:]) else 0) diff --git a/test/integration/targets/ansible-test-no-tty/ansible_collections/ns/col/tests/integration/targets/no-tty/aliases b/test/integration/targets/ansible-test-no-tty/ansible_collections/ns/col/tests/integration/targets/no-tty/aliases new file mode 100644 index 00000000..1af1cf90 --- /dev/null +++ b/test/integration/targets/ansible-test-no-tty/ansible_collections/ns/col/tests/integration/targets/no-tty/aliases @@ -0,0 +1 @@ +context/controller diff --git a/test/integration/targets/ansible-test-no-tty/ansible_collections/ns/col/tests/integration/targets/no-tty/assert-no-tty.py b/test/integration/targets/ansible-test-no-tty/ansible_collections/ns/col/tests/integration/targets/no-tty/assert-no-tty.py new file mode 100755 index 00000000..a2b094e2 --- /dev/null +++ b/test/integration/targets/ansible-test-no-tty/ansible_collections/ns/col/tests/integration/targets/no-tty/assert-no-tty.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +"""Assert no TTY is available.""" + +import sys + +status = 0 + +for handle in sys.stdin, sys.stdout, sys.stderr: + if handle.isatty(): + print(f'{handle} is a TTY', file=sys.stderr) + status += 1 + +sys.exit(status) diff --git a/test/integration/targets/ansible-test-no-tty/ansible_collections/ns/col/tests/integration/targets/no-tty/runme.sh b/test/integration/targets/ansible-test-no-tty/ansible_collections/ns/col/tests/integration/targets/no-tty/runme.sh new file mode 100755 index 00000000..ae712ddf --- /dev/null +++ b/test/integration/targets/ansible-test-no-tty/ansible_collections/ns/col/tests/integration/targets/no-tty/runme.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -eux + +./assert-no-tty.py diff --git a/test/integration/targets/ansible-test-no-tty/ansible_collections/ns/col/vendored_pty.py b/test/integration/targets/ansible-test-no-tty/ansible_collections/ns/col/vendored_pty.py new file mode 100644 index 00000000..bc70803b --- /dev/null +++ b/test/integration/targets/ansible-test-no-tty/ansible_collections/ns/col/vendored_pty.py @@ -0,0 +1,189 @@ +# Vendored copy of https://github.com/python/cpython/blob/3680ebed7f3e529d01996dd0318601f9f0d02b4b/Lib/pty.py +# PSF License (see licenses/PSF-license.txt or https://opensource.org/licenses/Python-2.0) +"""Pseudo terminal utilities.""" + +# Bugs: No signal handling. Doesn't set slave termios and window size. +# Only tested on Linux, FreeBSD, and macOS. +# See: W. Richard Stevens. 1992. Advanced Programming in the +# UNIX Environment. Chapter 19. +# Author: Steen Lumholt -- with additions by Guido. + +from select import select +import os +import sys +import tty + +# names imported directly for test mocking purposes +from os import close, waitpid +from tty import setraw, tcgetattr, tcsetattr + +__all__ = ["openpty", "fork", "spawn"] + +STDIN_FILENO = 0 +STDOUT_FILENO = 1 +STDERR_FILENO = 2 + +CHILD = 0 + +def openpty(): + """openpty() -> (master_fd, slave_fd) + Open a pty master/slave pair, using os.openpty() if possible.""" + + try: + return os.openpty() + except (AttributeError, OSError): + pass + master_fd, slave_name = _open_terminal() + slave_fd = slave_open(slave_name) + return master_fd, slave_fd + +def master_open(): + """master_open() -> (master_fd, slave_name) + Open a pty master and return the fd, and the filename of the slave end. + Deprecated, use openpty() instead.""" + + try: + master_fd, slave_fd = os.openpty() + except (AttributeError, OSError): + pass + else: + slave_name = os.ttyname(slave_fd) + os.close(slave_fd) + return master_fd, slave_name + + return _open_terminal() + +def _open_terminal(): + """Open pty master and return (master_fd, tty_name).""" + for x in 'pqrstuvwxyzPQRST': + for y in '0123456789abcdef': + pty_name = '/dev/pty' + x + y + try: + fd = os.open(pty_name, os.O_RDWR) + except OSError: + continue + return (fd, '/dev/tty' + x + y) + raise OSError('out of pty devices') + +def slave_open(tty_name): + """slave_open(tty_name) -> slave_fd + Open the pty slave and acquire the controlling terminal, returning + opened filedescriptor. + Deprecated, use openpty() instead.""" + + result = os.open(tty_name, os.O_RDWR) + try: + from fcntl import ioctl, I_PUSH + except ImportError: + return result + try: + ioctl(result, I_PUSH, "ptem") + ioctl(result, I_PUSH, "ldterm") + except OSError: + pass + return result + +def fork(): + """fork() -> (pid, master_fd) + Fork and make the child a session leader with a controlling terminal.""" + + try: + pid, fd = os.forkpty() + except (AttributeError, OSError): + pass + else: + if pid == CHILD: + try: + os.setsid() + except OSError: + # os.forkpty() already set us session leader + pass + return pid, fd + + master_fd, slave_fd = openpty() + pid = os.fork() + if pid == CHILD: + # Establish a new session. + os.setsid() + os.close(master_fd) + + # Slave becomes stdin/stdout/stderr of child. + os.dup2(slave_fd, STDIN_FILENO) + os.dup2(slave_fd, STDOUT_FILENO) + os.dup2(slave_fd, STDERR_FILENO) + if slave_fd > STDERR_FILENO: + os.close(slave_fd) + + # Explicitly open the tty to make it become a controlling tty. + tmp_fd = os.open(os.ttyname(STDOUT_FILENO), os.O_RDWR) + os.close(tmp_fd) + else: + os.close(slave_fd) + + # Parent and child process. + return pid, master_fd + +def _writen(fd, data): + """Write all the data to a descriptor.""" + while data: + n = os.write(fd, data) + data = data[n:] + +def _read(fd): + """Default read function.""" + return os.read(fd, 1024) + +def _copy(master_fd, master_read=_read, stdin_read=_read): + """Parent copy loop. + Copies + pty master -> standard output (master_read) + standard input -> pty master (stdin_read)""" + fds = [master_fd, STDIN_FILENO] + while fds: + rfds, _wfds, _xfds = select(fds, [], []) + + if master_fd in rfds: + # Some OSes signal EOF by returning an empty byte string, + # some throw OSErrors. + try: + data = master_read(master_fd) + except OSError: + data = b"" + if not data: # Reached EOF. + return # Assume the child process has exited and is + # unreachable, so we clean up. + else: + os.write(STDOUT_FILENO, data) + + if STDIN_FILENO in rfds: + data = stdin_read(STDIN_FILENO) + if not data: + fds.remove(STDIN_FILENO) + else: + _writen(master_fd, data) + +def spawn(argv, master_read=_read, stdin_read=_read): + """Create a spawned process.""" + if isinstance(argv, str): + argv = (argv,) + sys.audit('pty.spawn', argv) + + pid, master_fd = fork() + if pid == CHILD: + os.execlp(argv[0], *argv) + + try: + mode = tcgetattr(STDIN_FILENO) + setraw(STDIN_FILENO) + restore = True + except tty.error: # This is the same as termios.error + restore = False + + try: + _copy(master_fd, master_read, stdin_read) + finally: + if restore: + tcsetattr(STDIN_FILENO, tty.TCSAFLUSH, mode) + + close(master_fd) + return waitpid(pid, 0)[1] diff --git a/test/integration/targets/ansible-test-no-tty/runme.sh b/test/integration/targets/ansible-test-no-tty/runme.sh new file mode 100755 index 00000000..c02793a1 --- /dev/null +++ b/test/integration/targets/ansible-test-no-tty/runme.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Verify that ansible-test runs integration tests without a TTY. + +source ../collection/setup.sh + +set -x + +if ./run-with-pty.py tests/integration/targets/no-tty/assert-no-tty.py > /dev/null; then + echo "PTY assertion did not fail. Either PTY creation failed or PTY detection is broken." + exit 1 +fi + +./run-with-pty.py ansible-test integration --color "${@}" diff --git a/test/integration/targets/ansible-test-sanity-lint/aliases b/test/integration/targets/ansible-test-sanity-lint/aliases new file mode 100644 index 00000000..193276cc --- /dev/null +++ b/test/integration/targets/ansible-test-sanity-lint/aliases @@ -0,0 +1,4 @@ +shippable/posix/group1 # runs in the distro test containers +shippable/generic/group1 # runs in the default test container +context/controller +needs/target/collection diff --git a/test/integration/targets/ansible-test-sanity-lint/expected.txt b/test/integration/targets/ansible-test-sanity-lint/expected.txt new file mode 100644 index 00000000..94238c8a --- /dev/null +++ b/test/integration/targets/ansible-test-sanity-lint/expected.txt @@ -0,0 +1 @@ +plugins/modules/python-wrong-shebang.py:1:1: expected module shebang "b'#!/usr/bin/python'" but found: b'#!invalid' diff --git a/test/integration/targets/ansible-test-sanity-lint/runme.sh b/test/integration/targets/ansible-test-sanity-lint/runme.sh new file mode 100755 index 00000000..3e73cb4a --- /dev/null +++ b/test/integration/targets/ansible-test-sanity-lint/runme.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Make sure that `ansible-test sanity --lint` outputs the correct format to stdout, even when delegation is used. + +set -eu + +# Create test scenarios at runtime that do not pass sanity tests. +# This avoids the need to create ignore entries for the tests. + +mkdir -p ansible_collections/ns/col/plugins/modules + +( + cd ansible_collections/ns/col/plugins/modules + + echo '#!invalid' > python-wrong-shebang.py # expected module shebang "b'#!/usr/bin/python'" but found: b'#!invalid' +) + +source ../collection/setup.sh + +set -x + +### +### Run the sanity test with the `--lint` option. +### + +# Use the `--venv` option to verify that delegation preserves the output streams. +ansible-test sanity --test shebang --color --failure-ok --lint --venv "${@}" 1> actual-stdout.txt 2> actual-stderr.txt +diff -u "${TEST_DIR}/expected.txt" actual-stdout.txt +grep -f "${TEST_DIR}/expected.txt" actual-stderr.txt + +# Run without delegation to verify direct output uses the correct streams. +ansible-test sanity --test shebang --color --failure-ok --lint "${@}" 1> actual-stdout.txt 2> actual-stderr.txt +diff -u "${TEST_DIR}/expected.txt" actual-stdout.txt +grep -f "${TEST_DIR}/expected.txt" actual-stderr.txt + +### +### Run the sanity test without the `--lint` option. +### + +# Use the `--venv` option to verify that delegation preserves the output streams. +ansible-test sanity --test shebang --color --failure-ok --venv "${@}" 1> actual-stdout.txt 2> actual-stderr.txt +grep -f "${TEST_DIR}/expected.txt" actual-stdout.txt +[ ! -s actual-stderr.txt ] + +# Run without delegation to verify direct output uses the correct streams. +ansible-test sanity --test shebang --color --failure-ok "${@}" 1> actual-stdout.txt 2> actual-stderr.txt +grep -f "${TEST_DIR}/expected.txt" actual-stdout.txt +[ ! -s actual-stderr.txt ] diff --git a/test/integration/targets/ansible-test-sanity-shebang/aliases b/test/integration/targets/ansible-test-sanity-shebang/aliases new file mode 100644 index 00000000..193276cc --- /dev/null +++ b/test/integration/targets/ansible-test-sanity-shebang/aliases @@ -0,0 +1,4 @@ +shippable/posix/group1 # runs in the distro test containers +shippable/generic/group1 # runs in the default test container +context/controller +needs/target/collection diff --git a/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/plugins/modules/powershell.ps1 b/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/plugins/modules/powershell.ps1 new file mode 100644 index 00000000..9eb7192c --- /dev/null +++ b/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/plugins/modules/powershell.ps1 @@ -0,0 +1 @@ +#!powershell diff --git a/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/plugins/modules/python-no-shebang.py b/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/plugins/modules/python-no-shebang.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/plugins/modules/python-no-shebang.py diff --git a/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/plugins/modules/python.py b/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/plugins/modules/python.py new file mode 100644 index 00000000..013e4b7e --- /dev/null +++ b/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/plugins/modules/python.py @@ -0,0 +1 @@ +#!/usr/bin/python diff --git a/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/scripts/env_bash.sh b/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/scripts/env_bash.sh new file mode 100755 index 00000000..f1f641af --- /dev/null +++ b/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/scripts/env_bash.sh @@ -0,0 +1 @@ +#!/usr/bin/env bash diff --git a/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/scripts/env_python.py b/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/scripts/env_python.py new file mode 100755 index 00000000..4265cc3e --- /dev/null +++ b/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/scripts/env_python.py @@ -0,0 +1 @@ +#!/usr/bin/env python diff --git a/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/scripts/sh.sh b/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/scripts/sh.sh new file mode 100755 index 00000000..1a248525 --- /dev/null +++ b/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/scripts/sh.sh @@ -0,0 +1 @@ +#!/bin/sh diff --git a/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/tests/integration/targets/valid/env_bash.sh b/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/tests/integration/targets/valid/env_bash.sh new file mode 100755 index 00000000..f1f641af --- /dev/null +++ b/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/tests/integration/targets/valid/env_bash.sh @@ -0,0 +1 @@ +#!/usr/bin/env bash diff --git a/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/tests/integration/targets/valid/env_python.py b/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/tests/integration/targets/valid/env_python.py new file mode 100755 index 00000000..4265cc3e --- /dev/null +++ b/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/tests/integration/targets/valid/env_python.py @@ -0,0 +1 @@ +#!/usr/bin/env python diff --git a/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/tests/integration/targets/valid/sh.sh b/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/tests/integration/targets/valid/sh.sh new file mode 100755 index 00000000..1a248525 --- /dev/null +++ b/test/integration/targets/ansible-test-sanity-shebang/ansible_collections/ns/col/tests/integration/targets/valid/sh.sh @@ -0,0 +1 @@ +#!/bin/sh diff --git a/test/integration/targets/ansible-test-sanity-shebang/expected.txt b/test/integration/targets/ansible-test-sanity-shebang/expected.txt new file mode 100644 index 00000000..fbd73306 --- /dev/null +++ b/test/integration/targets/ansible-test-sanity-shebang/expected.txt @@ -0,0 +1,9 @@ +plugins/modules/no-shebang-executable.py:0:0: file without shebang should not be executable +plugins/modules/python-executable.py:0:0: module should not be executable +plugins/modules/python-wrong-shebang.py:1:1: expected module shebang "b'#!/usr/bin/python'" but found: b'#!invalid' +plugins/modules/utf-16-be-bom.py:0:0: file starts with a UTF-16 (BE) byte order mark +plugins/modules/utf-16-le-bom.py:0:0: file starts with a UTF-16 (LE) byte order mark +plugins/modules/utf-32-be-bom.py:0:0: file starts with a UTF-32 (BE) byte order mark +plugins/modules/utf-32-le-bom.py:0:0: file starts with a UTF-32 (LE) byte order mark +plugins/modules/utf-8-bom.py:0:0: file starts with a UTF-8 byte order mark +scripts/unexpected-shebang:1:1: unexpected non-module shebang: b'#!/usr/bin/custom' diff --git a/test/integration/targets/ansible-test-sanity-shebang/runme.sh b/test/integration/targets/ansible-test-sanity-shebang/runme.sh new file mode 100755 index 00000000..f7fc68a5 --- /dev/null +++ b/test/integration/targets/ansible-test-sanity-shebang/runme.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +set -eu + +# Create test scenarios at runtime that do not pass sanity tests. +# This avoids the need to create ignore entries for the tests. + +( + cd ansible_collections/ns/col/plugins/modules + + touch no-shebang-executable.py && chmod +x no-shebang-executable.py # file without shebang should not be executable + python -c "open('utf-32-be-bom.py', 'wb').write(b'\x00\x00\xFE\xFF')" # file starts with a UTF-32 (BE) byte order mark + python -c "open('utf-32-le-bom.py', 'wb').write(b'\xFF\xFE\x00\x00')" # file starts with a UTF-32 (LE) byte order mark + python -c "open('utf-16-be-bom.py', 'wb').write(b'\xFE\xFF')" # file starts with a UTF-16 (BE) byte order mark + python -c "open('utf-16-le-bom.py', 'wb').write(b'\xFF\xFE')" # file starts with a UTF-16 (LE) byte order mark + python -c "open('utf-8-bom.py', 'wb').write(b'\xEF\xBB\xBF')" # file starts with a UTF-8 byte order mark + echo '#!/usr/bin/python' > python-executable.py && chmod +x python-executable.py # module should not be executable + echo '#!invalid' > python-wrong-shebang.py # expected module shebang "b'#!/usr/bin/python'" but found: b'#!invalid' +) + +( + cd ansible_collections/ns/col/scripts + + echo '#!/usr/bin/custom' > unexpected-shebang # unexpected non-module shebang: b'#!/usr/bin/custom' + + echo '#!/usr/bin/make -f' > Makefile && chmod +x Makefile # pass + echo '#!/bin/bash -eu' > bash_eu.sh && chmod +x bash_eu.sh # pass + echo '#!/bin/bash -eux' > bash_eux.sh && chmod +x bash_eux.sh # pass + echo '#!/usr/bin/env fish' > env_fish.fish && chmod +x env_fish.fish # pass + echo '#!/usr/bin/env pwsh' > env_pwsh.ps1 && chmod +x env_pwsh.ps1 # pass +) + +mkdir ansible_collections/ns/col/examples + +( + cd ansible_collections/ns/col/examples + + echo '#!/usr/bin/custom' > unexpected-shebang # pass +) + +source ../collection/setup.sh + +set -x + +ansible-test sanity --test shebang --color --lint --failure-ok "${@}" > actual.txt + +diff -u "${TEST_DIR}/expected.txt" actual.txt diff --git a/test/integration/targets/ansible-test-shell/aliases b/test/integration/targets/ansible-test-shell/aliases new file mode 100644 index 00000000..193276cc --- /dev/null +++ b/test/integration/targets/ansible-test-shell/aliases @@ -0,0 +1,4 @@ +shippable/posix/group1 # runs in the distro test containers +shippable/generic/group1 # runs in the default test container +context/controller +needs/target/collection diff --git a/test/integration/targets/ansible-test-shell/ansible_collections/ns/col/.keep b/test/integration/targets/ansible-test-shell/ansible_collections/ns/col/.keep new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/test/integration/targets/ansible-test-shell/ansible_collections/ns/col/.keep diff --git a/test/integration/targets/ansible-test-shell/expected-stderr.txt b/test/integration/targets/ansible-test-shell/expected-stderr.txt new file mode 100644 index 00000000..af6415db --- /dev/null +++ b/test/integration/targets/ansible-test-shell/expected-stderr.txt @@ -0,0 +1 @@ +stderr diff --git a/test/integration/targets/ansible-test-shell/expected-stdout.txt b/test/integration/targets/ansible-test-shell/expected-stdout.txt new file mode 100644 index 00000000..faa3a15c --- /dev/null +++ b/test/integration/targets/ansible-test-shell/expected-stdout.txt @@ -0,0 +1 @@ +stdout diff --git a/test/integration/targets/ansible-test-shell/runme.sh b/test/integration/targets/ansible-test-shell/runme.sh new file mode 100755 index 00000000..0e0d18ae --- /dev/null +++ b/test/integration/targets/ansible-test-shell/runme.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Make sure that `ansible-test shell` outputs to the correct stream. + +set -eu + +source ../collection/setup.sh + +set -x + +# Try `shell` with delegation. + +ansible-test shell --venv -- \ + python -c 'import sys; print("stdout"); print("stderr", file=sys.stderr)' 1> actual-stdout.txt 2> actual-stderr.txt + +cat actual-stdout.txt +cat actual-stderr.txt + +diff -u "${TEST_DIR}/expected-stdout.txt" actual-stdout.txt +grep -f "${TEST_DIR}/expected-stderr.txt" actual-stderr.txt + +# Try `shell` without delegation. + +ansible-test shell -- \ + python -c 'import sys; print("stdout"); print("stderr", file=sys.stderr)' 1> actual-stdout.txt 2> actual-stderr.txt + +cat actual-stdout.txt +cat actual-stderr.txt + +diff -u "${TEST_DIR}/expected-stdout.txt" actual-stdout.txt +grep -f "${TEST_DIR}/expected-stderr.txt" actual-stderr.txt diff --git a/test/integration/targets/ansible-test/aliases b/test/integration/targets/ansible-test/aliases index b98e7bb2..002fe2cf 100644 --- a/test/integration/targets/ansible-test/aliases +++ b/test/integration/targets/ansible-test/aliases @@ -1,4 +1,5 @@ shippable/posix/group1 # runs in the distro test containers shippable/generic/group1 # runs in the default test container context/controller +needs/target/collection destructive # adds and then removes packages into lib/ansible/_vendor/ diff --git a/test/integration/targets/ansible-test/collection-tests/coverage.sh b/test/integration/targets/ansible-test/collection-tests/coverage.sh index c2336a32..ddc0f9b4 100755 --- a/test/integration/targets/ansible-test/collection-tests/coverage.sh +++ b/test/integration/targets/ansible-test/collection-tests/coverage.sh @@ -5,7 +5,7 @@ set -eux -o pipefail cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}" cd "${WORK_DIR}/ansible_collections/ns/col" -"${TEST_DIR}/collection-tests/update-ignore.py" +"${TEST_DIR}/../collection/update-ignore.py" # common args for all tests common=(--venv --color --truncate 0 "${@}") diff --git a/test/integration/targets/ansible-test/collection-tests/sanity-vendor.sh b/test/integration/targets/ansible-test/collection-tests/sanity-vendor.sh index 0fcd659b..72043bfd 100755 --- a/test/integration/targets/ansible-test/collection-tests/sanity-vendor.sh +++ b/test/integration/targets/ansible-test/collection-tests/sanity-vendor.sh @@ -5,7 +5,7 @@ set -eux -o pipefail cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}" cd "${WORK_DIR}/ansible_collections/ns/col" -"${TEST_DIR}/collection-tests/update-ignore.py" +"${TEST_DIR}/../collection/update-ignore.py" vendor_dir="$(python -c 'import pathlib, ansible._vendor; print(pathlib.Path(ansible._vendor.__file__).parent)')" diff --git a/test/integration/targets/ansible-test/collection-tests/sanity.sh b/test/integration/targets/ansible-test/collection-tests/sanity.sh index 21e8607b..99d9b427 100755 --- a/test/integration/targets/ansible-test/collection-tests/sanity.sh +++ b/test/integration/targets/ansible-test/collection-tests/sanity.sh @@ -5,6 +5,6 @@ set -eux -o pipefail cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}" cd "${WORK_DIR}/ansible_collections/ns/col" -"${TEST_DIR}/collection-tests/update-ignore.py" +"${TEST_DIR}/../collection/update-ignore.py" ansible-test sanity --color --truncate 0 "${@}" diff --git a/test/integration/targets/apt/tasks/apt.yml b/test/integration/targets/apt/tasks/apt.yml index 81614118..5b1a24a3 100644 --- a/test/integration/targets/apt/tasks/apt.yml +++ b/test/integration/targets/apt/tasks/apt.yml @@ -507,3 +507,25 @@ that: - "allow_change_held_packages_no_update is not changed" - "allow_change_held_packages_hello_version.stdout == allow_change_held_packages_hello_version_again.stdout" + +# Virtual package +- name: Install a virtual package + apt: + package: + - emacs-nox + - yaml-mode # <- the virtual package + state: latest + register: install_virtual_package_result + +- name: Check the virtual package install result + assert: + that: + - install_virtual_package_result is changed + +- name: Clean up virtual-package install + apt: + package: + - emacs-nox + - elpa-yaml-mode + state: absent + purge: yes diff --git a/test/integration/targets/collection/aliases b/test/integration/targets/collection/aliases new file mode 100644 index 00000000..136c05e0 --- /dev/null +++ b/test/integration/targets/collection/aliases @@ -0,0 +1 @@ +hidden diff --git a/test/integration/targets/collection/setup.sh b/test/integration/targets/collection/setup.sh new file mode 100755 index 00000000..f1b33a55 --- /dev/null +++ b/test/integration/targets/collection/setup.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Source this file from collection integration tests. +# +# It simplifies several aspects of collection testing: +# +# 1) Collection tests must be executed outside of the ansible source tree. +# Otherwise ansible-test will test the ansible source instead of the test collection. +# The temporary directory provided by ansible-test resides within the ansible source tree. +# +# 2) Sanity test ignore files for collections must be versioned based on the ansible-core version being used. +# This script generates an ignore file with the correct filename for the current ansible-core version. +# +# 3) Sanity tests which are multi-version require an ignore entry per Python version. +# This script replicates these ignore entries for each supported Python version based on the ignored path. + +set -eu -o pipefail + +export TEST_DIR +export WORK_DIR + +TEST_DIR="$PWD" +WORK_DIR="$(mktemp -d)" + +trap 'rm -rf "${WORK_DIR}"' EXIT + +cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}" +cd "${WORK_DIR}/ansible_collections/ns/col" + +"${TEST_DIR}/../collection/update-ignore.py" diff --git a/test/integration/targets/ansible-test/collection-tests/update-ignore.py b/test/integration/targets/collection/update-ignore.py index 51ddf9ac..92a702cf 100755 --- a/test/integration/targets/ansible-test/collection-tests/update-ignore.py +++ b/test/integration/targets/collection/update-ignore.py @@ -16,6 +16,11 @@ def main(): from ansible_test._internal import constants src_path = 'tests/sanity/ignore.txt' + + if not os.path.exists(src_path): + print(f'Skipping updates on non-existent ignore file: {src_path}') + return + directory = os.path.dirname(src_path) name, ext = os.path.splitext(os.path.basename(src_path)) major_minor = '.'.join(release.__version__.split('.')[:2]) diff --git a/test/integration/targets/config/runme.sh b/test/integration/targets/config/runme.sh index 76df44c4..122e15d7 100755 --- a/test/integration/targets/config/runme.sh +++ b/test/integration/targets/config/runme.sh @@ -7,7 +7,7 @@ set -eux ANSIBLE_TIMEOUT= ansible -m ping testhost -i ../../inventory "$@" # env var is wrong type, this should be a fatal error pointing at the setting -ANSIBLE_TIMEOUT='lola' ansible -m ping testhost -i ../../inventory "$@" 2>&1|grep 'Invalid type for configuration option setting: DEFAULT_TIMEOUT' +ANSIBLE_TIMEOUT='lola' ansible -m ping testhost -i ../../inventory "$@" 2>&1|grep 'Invalid type for configuration option setting: DEFAULT_TIMEOUT (from env: ANSIBLE_TIMEOUT)' # https://github.com/ansible/ansible/issues/69577 ANSIBLE_REMOTE_TMP="$HOME/.ansible/directory_with_no_space" ansible -m ping testhost -i ../../inventory "$@" @@ -19,6 +19,10 @@ ANSIBLE_CONFIG=nonexistent.cfg ansible-config dump --only-changed -v | grep 'No # https://github.com/ansible/ansible/pull/73715 ANSIBLE_CONFIG=inline_comment_ansible.cfg ansible-config dump --only-changed | grep "'ansibull'" +# test type headers are only displayed with --only-changed -t all for changed options +env -i PATH="$PATH" PYTHONPATH="$PYTHONPATH" ansible-config dump --only-changed -t all | grep -v "CONNECTION" +env -i PATH="$PATH" PYTHONPATH="$PYTHONPATH" ANSIBLE_SSH_PIPELINING=True ansible-config dump --only-changed -t all | grep "CONNECTION" + # test the config option validation ansible-playbook validation.yml "$@" diff --git a/test/integration/targets/connection_ssh/runme.sh b/test/integration/targets/connection_ssh/runme.sh index 4d430263..ad817c83 100755 --- a/test/integration/targets/connection_ssh/runme.sh +++ b/test/integration/targets/connection_ssh/runme.sh @@ -46,14 +46,18 @@ fi set -e -# temporary work-around for issues due to new scp filename checking -# https://github.com/ansible/ansible/issues/52640 -if [[ "$(scp -T 2>&1)" == "usage: scp "* ]]; then +if [[ "$(scp -O 2>&1)" == "usage: scp "* ]]; then + # scp supports the -O option (and thus the -T option as well) + # work-around required + # see: https://www.openssh.com/txt/release-9.0 + scp_args=("-e" "ansible_scp_extra_args=-TO") +elif [[ "$(scp -T 2>&1)" == "usage: scp "* ]]; then # scp supports the -T option # work-around required + # see: https://github.com/ansible/ansible/issues/52640 scp_args=("-e" "ansible_scp_extra_args=-T") else - # scp does not support the -T option + # scp does not support the -T or -O options # no work-around required # however we need to put something in the array to keep older versions of bash happy scp_args=("-e" "") diff --git a/test/integration/targets/dnf/tasks/main.yml b/test/integration/targets/dnf/tasks/main.yml index 591dc33a..96e5cbfa 100644 --- a/test/integration/targets/dnf/tasks/main.yml +++ b/test/integration/targets/dnf/tasks/main.yml @@ -54,8 +54,10 @@ - ansible_distribution_major_version is version('23', '>=') - include_tasks: modularity.yml - when: (ansible_distribution == 'Fedora' and ansible_distribution_major_version is version('29', '>=')) or - (ansible_distribution in ['RedHat', 'CentOS'] and ansible_distribution_major_version is version('8', '>=')) + when: + - astream_name is defined + - (ansible_distribution == 'Fedora' and ansible_distribution_major_version is version('29', '>=')) or + (ansible_distribution in ['RedHat', 'CentOS'] and ansible_distribution_major_version is version('8', '>=')) tags: - dnf_modularity diff --git a/test/integration/targets/dnf/vars/RedHat-9.yml b/test/integration/targets/dnf/vars/RedHat-9.yml index e700a9b9..5681e701 100644 --- a/test/integration/targets/dnf/vars/RedHat-9.yml +++ b/test/integration/targets/dnf/vars/RedHat-9.yml @@ -1,2 +1,3 @@ -astream_name: '@container-tools:latest/common' -astream_name_no_stream: '@container-tools/common' +# RHEL9.0 contains no modules, to be re-introduced in 9.1 +# astream_name: '@container-tools:latest/common' +# astream_name_no_stream: '@container-tools/common' diff --git a/test/integration/targets/group/tasks/tests.yml b/test/integration/targets/group/tasks/tests.yml index a724c9df..aaf7402d 100644 --- a/test/integration/targets/group/tasks/tests.yml +++ b/test/integration/targets/group/tasks/tests.yml @@ -211,7 +211,7 @@ - user_test_local_mode - name: Ensure lgroupadd is present - Alpine - command: apk add -U libuser --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community + command: apk add -U libuser when: ansible_distribution == 'Alpine' tags: - user_test_local_mode diff --git a/test/integration/targets/hostname/tasks/test_normal.yml b/test/integration/targets/hostname/tasks/test_normal.yml index ed5ac735..9534d73b 100644 --- a/test/integration/targets/hostname/tasks/test_normal.yml +++ b/test/integration/targets/hostname/tasks/test_normal.yml @@ -1,5 +1,5 @@ - name: Ensure hostname doesn't confuse NetworkManager - when: ansible_os_family == 'RedHat' + when: ansible_os_family == 'RedHat' and ansible_distribution_major_version is version('8') block: - name: slurp /var/log/messages slurp: @@ -23,7 +23,7 @@ register: current_after_hn2 - name: Ensure hostname doesn't confuse NetworkManager - when: ansible_os_family == 'RedHat' + when: ansible_os_family == 'RedHat' and ansible_distribution_major_version is version('8') block: - name: slurp /var/log/messages slurp: diff --git a/test/integration/targets/lookup_password/tasks/main.yml b/test/integration/targets/lookup_password/tasks/main.yml index 4eeef151..dacf032d 100644 --- a/test/integration/targets/lookup_password/tasks/main.yml +++ b/test/integration/targets/lookup_password/tasks/main.yml @@ -102,3 +102,48 @@ assert: that: - "newpass != newpass2" + +- name: test both types of args and that seed guarantees same results + vars: + pns: "{{passwords_noseed['results']}}" + inl: "{{passwords_inline['results']}}" + kv: "{{passwords['results']}}" + l: [1, 2, 3] + block: + - name: generate passwords w/o seed + debug: + msg: '{{ lookup("password", "/dev/null")}}' + loop: "{{ l }}" + register: passwords_noseed + + - name: verify they are all different, this is not guaranteed, but statisically almost impossible + assert: + that: + - pns[0]['msg'] != pns[1]['msg'] + - pns[0]['msg'] != pns[2]['msg'] + - pns[1]['msg'] != pns[2]['msg'] + + - name: generate passwords, with seed inline + debug: + msg: '{{ lookup("password", "/dev/null seed=foo")}}' + loop: "{{ l }}" + register: passwords_inline + + - name: verify they are all the same + assert: + that: + - inl[0]['msg'] == inl[1]['msg'] + - inl[0]['msg'] == inl[2]['msg'] + + - name: generate passwords, with seed k=v + debug: + msg: '{{ lookup("password", "/dev/null", seed="foo")}}' + loop: "{{ l }}" + register: passwords + + - name: verify they are all the same + assert: + that: + - kv[0]['msg'] == kv[1]['msg'] + - kv[0]['msg'] == kv[2]['msg'] + - kv[0]['msg'] == inl[0]['msg'] diff --git a/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/meta/runtime.yml b/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/meta/runtime.yml index 081ee8c2..a8c2c8c5 100644 --- a/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/meta/runtime.yml +++ b/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/meta/runtime.yml @@ -1,3 +1,49 @@ +plugin_routing: + action: + # Backwards compat for modules-redirected-as-actions: + # By default, each module_defaults entry is resolved as an action plugin, + # and if it does not exist, it is resolved a a module. + # All modules that redirect to the same action will resolve to the same action. + module_uses_action_defaults: + redirect: testns.testcoll.eos + + # module-redirected-as-action overridden by action_plugin + iosfacts: + redirect: testns.testcoll.nope + ios_facts: + redirect: testns.testcoll.nope + + redirected_action: + redirect: testns.testcoll.ios + modules: + # Any module_defaults for testns.testcoll.module will not apply to a module_uses_action_defaults task: + # + # module_defaults: + # testns.testcoll.module: + # option: value + # + # But defaults for testns.testcoll.module_uses_action_defaults or testns.testcoll.eos will: + # + # module_defaults: + # testns.testcoll.module_uses_action_defaults: + # option: value + # testns.testcoll.eos: + # option: defined_last_i_win + module_uses_action_defaults: + redirect: testns.testcoll.module + + # Not "eos_facts" to ensure TE is not finding handler via prefix + # eosfacts tasks should not get eos module_defaults (or defaults for other modules that use eos action plugin) + eosfacts: + action_plugin: testns.testcoll.eos + + # Test that `action_plugin` has higher precedence than module-redirected-as-action - reverse this? + # Current behavior is iosfacts/ios_facts do not get ios defaults. + iosfacts: + redirect: testns.testcoll.ios_facts + ios_facts: + action_plugin: testns.testcoll.redirected_action + action_groups: testgroup: # Test metadata 'extend_group' feature does not get stuck in a recursive loop diff --git a/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/plugins/action/eos.py b/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/plugins/action/eos.py new file mode 100644 index 00000000..0d39f26d --- /dev/null +++ b/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/plugins/action/eos.py @@ -0,0 +1,18 @@ +# Copyright: (c) 2022, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.plugins.action.normal import ActionModule as ActionBase +from ansible.utils.vars import merge_hash + + +class ActionModule(ActionBase): + + def run(self, tmp=None, task_vars=None): + + result = super(ActionModule, self).run(tmp, task_vars) + result['action_plugin'] = 'eos' + + return result diff --git a/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/plugins/action/ios.py b/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/plugins/action/ios.py new file mode 100644 index 00000000..20284fd1 --- /dev/null +++ b/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/plugins/action/ios.py @@ -0,0 +1,18 @@ +# Copyright: (c) 2022, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.plugins.action.normal import ActionModule as ActionBase +from ansible.utils.vars import merge_hash + + +class ActionModule(ActionBase): + + def run(self, tmp=None, task_vars=None): + + result = super(ActionModule, self).run(tmp, task_vars) + result['action_plugin'] = 'ios' + + return result diff --git a/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/plugins/action/vyos.py b/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/plugins/action/vyos.py new file mode 100644 index 00000000..b0e1904b --- /dev/null +++ b/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/plugins/action/vyos.py @@ -0,0 +1,18 @@ +# Copyright: (c) 2022, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.plugins.action.normal import ActionModule as ActionBase +from ansible.utils.vars import merge_hash + + +class ActionModule(ActionBase): + + def run(self, tmp=None, task_vars=None): + + result = super(ActionModule, self).run(tmp, task_vars) + result['action_plugin'] = 'vyos' + + return result diff --git a/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/plugins/modules/eosfacts.py b/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/plugins/modules/eosfacts.py new file mode 100644 index 00000000..8c73fe15 --- /dev/null +++ b/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/plugins/modules/eosfacts.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +# Copyright: (c) 2022, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + + +DOCUMENTATION = r''' +--- +module: eosfacts +short_description: module to test module_defaults +description: module to test module_defaults +version_added: '2.13' +''' + +EXAMPLES = r''' +''' + +from ansible.module_utils.basic import AnsibleModule + + +def main(): + module = AnsibleModule( + argument_spec=dict( + eosfacts=dict(type=bool), + ), + supports_check_mode=True + ) + module.exit_json(eosfacts=module.params['eosfacts']) + + +if __name__ == '__main__': + main() diff --git a/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/plugins/modules/ios_facts.py b/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/plugins/modules/ios_facts.py new file mode 100644 index 00000000..e2ed5981 --- /dev/null +++ b/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/plugins/modules/ios_facts.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +# Copyright: (c) 2022, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + + +DOCUMENTATION = r''' +--- +module: ios_facts +short_description: module to test module_defaults +description: module to test module_defaults +version_added: '2.13' +''' + +EXAMPLES = r''' +''' + +from ansible.module_utils.basic import AnsibleModule + + +def main(): + module = AnsibleModule( + argument_spec=dict( + ios_facts=dict(type=bool), + ), + supports_check_mode=True + ) + module.exit_json(ios_facts=module.params['ios_facts']) + + +if __name__ == '__main__': + main() diff --git a/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/plugins/modules/module.py b/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/plugins/modules/module.py new file mode 100644 index 00000000..b98a5f94 --- /dev/null +++ b/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/plugins/modules/module.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +# Copyright: (c) 2022, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + + +DOCUMENTATION = r''' +--- +module: module +short_description: module to test module_defaults +description: module to test module_defaults +version_added: '2.13' +''' + +EXAMPLES = r''' +''' + +from ansible.module_utils.basic import AnsibleModule + + +def main(): + module = AnsibleModule( + argument_spec=dict( + action_option=dict(type=bool), + ), + supports_check_mode=True + ) + module.exit_json(action_option=module.params['action_option']) + + +if __name__ == '__main__': + main() diff --git a/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/plugins/modules/vyosfacts.py b/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/plugins/modules/vyosfacts.py new file mode 100644 index 00000000..3a9abbc6 --- /dev/null +++ b/test/integration/targets/module_defaults/collections/ansible_collections/testns/testcoll/plugins/modules/vyosfacts.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +# Copyright: (c) 2022, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import absolute_import, division, print_function +__metaclass__ = type + + +DOCUMENTATION = r''' +--- +module: vyosfacts +short_description: module to test module_defaults +description: module to test module_defaults +version_added: '2.13' +''' + +EXAMPLES = r''' +''' + +from ansible.module_utils.basic import AnsibleModule + + +def main(): + module = AnsibleModule( + argument_spec=dict( + vyosfacts=dict(type=bool), + ), + supports_check_mode=True + ) + module.exit_json(vyosfacts=module.params['vyosfacts']) + + +if __name__ == '__main__': + main() diff --git a/test/integration/targets/module_defaults/runme.sh b/test/integration/targets/module_defaults/runme.sh index 082f4e5b..fe9c40ce 100755 --- a/test/integration/targets/module_defaults/runme.sh +++ b/test/integration/targets/module_defaults/runme.sh @@ -2,8 +2,13 @@ set -eux +# Symlink is test for backwards-compat (only workaround for https://github.com/ansible/ansible/issues/77059) +sudo ln -s "${PWD}/collections/ansible_collections/testns/testcoll/plugins/action/vyos.py" ./collections/ansible_collections/testns/testcoll/plugins/action/vyosfacts.py + ansible-playbook test_defaults.yml "$@" +sudo rm ./collections/ansible_collections/testns/testcoll/plugins/action/vyosfacts.py + ansible-playbook test_action_groups.yml "$@" ansible-playbook test_action_group_metadata.yml "$@" diff --git a/test/integration/targets/module_defaults/test_defaults.yml b/test/integration/targets/module_defaults/test_defaults.yml index 70377f12..6206d3a9 100644 --- a/test/integration/targets/module_defaults/test_defaults.yml +++ b/test/integration/targets/module_defaults/test_defaults.yml @@ -110,3 +110,140 @@ - "builtin_legacy_defaults_2.msg == 'legacy default'" - include_tasks: tasks/main.yml + +- name: test preferring module name defaults for platform-specific actions + hosts: localhost + gather_facts: no + tasks: + - name: ensure eosfacts does not use action plugin default + testns.testcoll.eosfacts: + module_defaults: + testns.testcoll.eos: + fail: true + + - name: eosfacts does use module name defaults + testns.testcoll.eosfacts: + module_defaults: + testns.testcoll.eosfacts: + eosfacts: true + register: result + + - assert: + that: + - result.eosfacts + - result.action_plugin == 'eos' + + - name: ensure vyosfacts does not use action plugin default + testns.testcoll.vyosfacts: + module_defaults: + testns.testcoll.vyos: + fail: true + + - name: vyosfacts does use vyosfacts defaults + testns.testcoll.vyosfacts: + module_defaults: + testns.testcoll.vyosfacts: + vyosfacts: true + register: result + + - assert: + that: + - result.vyosfacts + - result.action_plugin == 'vyos' + + - name: iosfacts/ios_facts does not use action plugin default (module action_plugin field has precedence over module-as-action-redirect) + collections: + - testns.testcoll + module_defaults: + testns.testcoll.ios: + fail: true + block: + - ios_facts: + register: result + - assert: + that: + - result.action_plugin == 'ios' + + - iosfacts: + register: result + - assert: + that: + - result.action_plugin == 'ios' + + - name: ensure iosfacts/ios_facts uses ios_facts defaults + collections: + - testns.testcoll + module_defaults: + testns.testcoll.ios_facts: + ios_facts: true + block: + - ios_facts: + register: result + - assert: + that: + - result.ios_facts + - result.action_plugin == 'ios' + + - iosfacts: + register: result + - assert: + that: + - result.ios_facts + - result.action_plugin == 'ios' + + - name: ensure iosfacts/ios_facts uses iosfacts defaults + collections: + - testns.testcoll + module_defaults: + testns.testcoll.iosfacts: + ios_facts: true + block: + - ios_facts: + register: result + - assert: + that: + - result.ios_facts + - result.action_plugin == 'ios' + + - iosfacts: + register: result + - assert: + that: + - result.ios_facts + - result.action_plugin == 'ios' + + - name: ensure redirected action gets redirected action defaults + testns.testcoll.module_uses_action_defaults: + module_defaults: + testns.testcoll.module_uses_action_defaults: + action_option: true + register: result + + - assert: + that: + - result.action_option + - result.action_plugin == 'eos' + + - name: ensure redirected action gets resolved action defaults + testns.testcoll.module_uses_action_defaults: + module_defaults: + testns.testcoll.eos: + action_option: true + register: result + + - assert: + that: + - result.action_option + - result.action_plugin == 'eos' + + - name: ensure redirected action does not use module-specific defaults + testns.testcoll.module_uses_action_defaults: + module_defaults: + testns.testcoll.module: + fail: true + register: result + + - assert: + that: + - not result.action_option + - result.action_plugin == 'eos' diff --git a/test/integration/targets/module_utils_facts.system.selinux/aliases b/test/integration/targets/module_utils_facts.system.selinux/aliases index ee281d27..a6dafcf8 100644 --- a/test/integration/targets/module_utils_facts.system.selinux/aliases +++ b/test/integration/targets/module_utils_facts.system.selinux/aliases @@ -1,5 +1 @@ shippable/posix/group1 -skip/osx -skip/macos -skip/freebsd -skip/docker diff --git a/test/integration/targets/module_utils_facts.system.selinux/tasks/main.yml b/test/integration/targets/module_utils_facts.system.selinux/tasks/main.yml index c599377b..17172395 100644 --- a/test/integration/targets/module_utils_facts.system.selinux/tasks/main.yml +++ b/test/integration/targets/module_utils_facts.system.selinux/tasks/main.yml @@ -18,11 +18,12 @@ - name: check selinux policy type shell: grep '^SELINUXTYPE=' /etc/selinux/config | cut -d'=' -f2 + ignore_errors: yes register: r - set_fact: selinux_policytype: "{{ r.stdout_lines[0] }}" - when: r.changed + when: r is success and r.stdout_lines - assert: that: diff --git a/test/integration/targets/plugin_loader/normal/action_plugins/self_referential.py b/test/integration/targets/plugin_loader/normal/action_plugins/self_referential.py new file mode 100644 index 00000000..b4c89577 --- /dev/null +++ b/test/integration/targets/plugin_loader/normal/action_plugins/self_referential.py @@ -0,0 +1,29 @@ +# Copyright: Contributors to the Ansible project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from ansible.plugins.action import ActionBase + +import sys + +# reference our own module from sys.modules while it's being loaded to ensure the importer behaves properly +try: + mod = sys.modules[__name__] +except KeyError: + raise Exception(f'module {__name__} is not accessible via sys.modules, likely a pluginloader bug') + + +class ActionModule(ActionBase): + TRANSFERS_FILES = False + + def run(self, tmp=None, task_vars=None): + if task_vars is None: + task_vars = dict() + + result = super(ActionModule, self).run(tmp, task_vars) + del tmp # tmp no longer has any effect + + result['changed'] = False + result['msg'] = 'self-referential action loaded and ran successfully' + return result diff --git a/test/integration/targets/plugin_loader/normal/self_referential.yml b/test/integration/targets/plugin_loader/normal/self_referential.yml new file mode 100644 index 00000000..d3eed218 --- /dev/null +++ b/test/integration/targets/plugin_loader/normal/self_referential.yml @@ -0,0 +1,5 @@ +- hosts: testhost + gather_facts: false + tasks: + - name: ensure a self-referential action plugin loads properly + self_referential: diff --git a/test/integration/targets/plugin_loader/runme.sh b/test/integration/targets/plugin_loader/runme.sh index 8ce7803a..e30f6241 100755 --- a/test/integration/targets/plugin_loader/runme.sh +++ b/test/integration/targets/plugin_loader/runme.sh @@ -31,3 +31,6 @@ do exit 1 fi done + +# test config loading +ansible-playbook use_coll_name.yml -i ../../inventory -e 'ansible_connection=ansible.builtin.ssh' "$@" diff --git a/test/integration/targets/plugin_loader/use_coll_name.yml b/test/integration/targets/plugin_loader/use_coll_name.yml new file mode 100644 index 00000000..66507ced --- /dev/null +++ b/test/integration/targets/plugin_loader/use_coll_name.yml @@ -0,0 +1,7 @@ +- name: ensure configuration is loaded when we use FQCN and have already loaded using 'short namne' (which is case will all builtin connection plugins) + hosts: all + gather_facts: false + tasks: + - name: relies on extra var being passed in with connection and fqcn + ping: + ignore_unreachable: True diff --git a/test/integration/targets/rpm_key/tasks/rpm_key.yaml b/test/integration/targets/rpm_key/tasks/rpm_key.yaml index 24fbbaee..89ed2361 100644 --- a/test/integration/targets/rpm_key/tasks/rpm_key.yaml +++ b/test/integration/targets/rpm_key/tasks/rpm_key.yaml @@ -29,11 +29,6 @@ url: https://ci-files.testing.ansible.com/test/integration/targets/rpm_key/sl-5.02-1.el7.x86_64.rpm dest: /tmp/sl.rpm -- name: download Mono key - get_url: - url: https://ci-files.testing.ansible.com/test/integration/targets/rpm_key/mono.gpg - dest: /tmp/mono.gpg - - name: remove EPEL GPG key from keyring rpm_key: state: absent @@ -69,21 +64,11 @@ rpm_key: state: present key: /tmp/RPM-GPG-KEY-EPEL-7 - -- name: add Mono gpg key - rpm_key: - state: present - key: /tmp/mono.gpg - -- name: add Mono gpg key - rpm_key: - state: present - key: /tmp/mono.gpg - register: mono_indempotence + register: key_idempotence - name: verify idempotence assert: - that: "not mono_indempotence.changed" + that: "not key_idempotence.changed" - name: check GPG signature of sl. Should return okay shell: "rpm --checksig /tmp/sl.rpm" diff --git a/test/integration/targets/setup_cron/tasks/main.yml b/test/integration/targets/setup_cron/tasks/main.yml index d7ce3303..7005ce6b 100644 --- a/test/integration/targets/setup_cron/tasks/main.yml +++ b/test/integration/targets/setup_cron/tasks/main.yml @@ -27,6 +27,11 @@ when: ansible_distribution != 'Alpine' - name: install faketime packages - Alpine + # NOTE: The `faketime` package is currently only available in the + # NOTE: `edge` branch. + # FIXME: If it ever becomes available in the `main` repository for + # FIXME: currently tested Alpine versions, the `--repository=...` + # FIXME: option can be dropped. command: apk add -U {{ faketime_pkg }} --repository=http://dl-cdn.alpinelinux.org/alpine/edge/testing when: ansible_distribution == 'Alpine' diff --git a/test/integration/targets/template/runme.sh b/test/integration/targets/template/runme.sh index 78f8d7b5..a613c2a7 100755 --- a/test/integration/targets/template/runme.sh +++ b/test/integration/targets/template/runme.sh @@ -40,3 +40,5 @@ ansible-playbook unsafe.yml -v "$@" # ensure Jinja2 overrides from a template are used ansible-playbook in_template_overrides.yml -v "$@" + +ansible-playbook undefined_in_import.yml -i ../../inventory -v "$@" diff --git a/test/integration/targets/template/tasks/main.yml b/test/integration/targets/template/tasks/main.yml index e8a2b9a8..04919f18 100644 --- a/test/integration/targets/template/tasks/main.yml +++ b/test/integration/targets/template/tasks/main.yml @@ -761,3 +761,19 @@ - test vars: test: "{{ lookup('file', '{{ output_dir }}/empty_template.templated')|length == 0 }}" + +- assert: + that: + - data_not_converted | type_debug == 'NativeJinjaUnsafeText' + - data_converted | type_debug == 'dict' + vars: + data_not_converted: "{{ lookup('template', 'json_macro.j2', convert_data=False) }}" + data_converted: "{{ lookup('template', 'json_macro.j2') }}" + +- name: Test convert_data is correctly set to True for nested vars evaluation + debug: + msg: "{{ lookup('template', 'indirect_dict.j2', convert_data=False) }}" + vars: + d: + foo: bar + v: "{{ d }}" diff --git a/test/integration/targets/template/templates/indirect_dict.j2 b/test/integration/targets/template/templates/indirect_dict.j2 new file mode 100644 index 00000000..3124371f --- /dev/null +++ b/test/integration/targets/template/templates/indirect_dict.j2 @@ -0,0 +1 @@ +{{ v.foo }} diff --git a/test/integration/targets/template/templates/json_macro.j2 b/test/integration/targets/template/templates/json_macro.j2 new file mode 100644 index 00000000..080f1648 --- /dev/null +++ b/test/integration/targets/template/templates/json_macro.j2 @@ -0,0 +1,2 @@ +{% macro m() %}{{ {"foo":"bar"} }}{% endmacro %} +{{ m() }} diff --git a/test/integration/targets/template/undefined_in_import-import.j2 b/test/integration/targets/template/undefined_in_import-import.j2 new file mode 100644 index 00000000..fbb97b0d --- /dev/null +++ b/test/integration/targets/template/undefined_in_import-import.j2 @@ -0,0 +1 @@ +{{ undefined_variable }} diff --git a/test/integration/targets/template/undefined_in_import.j2 b/test/integration/targets/template/undefined_in_import.j2 new file mode 100644 index 00000000..619e4f70 --- /dev/null +++ b/test/integration/targets/template/undefined_in_import.j2 @@ -0,0 +1 @@ +{% import 'undefined_in_import-import.j2' as t %} diff --git a/test/integration/targets/template/undefined_in_import.yml b/test/integration/targets/template/undefined_in_import.yml new file mode 100644 index 00000000..62f60d66 --- /dev/null +++ b/test/integration/targets/template/undefined_in_import.yml @@ -0,0 +1,11 @@ +- hosts: localhost + gather_facts: false + tasks: + - debug: + msg: "{{ lookup('template', 'undefined_in_import.j2') }}" + ignore_errors: true + register: res + + - assert: + that: + - "\"'undefined_variable' is undefined\" in res.msg" diff --git a/test/integration/targets/templating_lookups/runme.sh b/test/integration/targets/templating_lookups/runme.sh index b900c153..60b3923b 100755 --- a/test/integration/targets/templating_lookups/runme.sh +++ b/test/integration/targets/templating_lookups/runme.sh @@ -2,7 +2,7 @@ set -eux -ANSIBLE_ROLES_PATH=./ UNICODE_VAR=café ansible-playbook runme.yml "$@" +ANSIBLE_LOOKUP_PLUGINS=. ANSIBLE_ROLES_PATH=./ UNICODE_VAR=café ansible-playbook runme.yml "$@" ansible-playbook template_lookup_vaulted/playbook.yml --vault-password-file template_lookup_vaulted/test_vault_pass "$@" diff --git a/test/integration/targets/templating_lookups/template_lookups/mock_lookup_plugins/77788.py b/test/integration/targets/templating_lookups/template_lookups/mock_lookup_plugins/77788.py new file mode 100644 index 00000000..436ceaf3 --- /dev/null +++ b/test/integration/targets/templating_lookups/template_lookups/mock_lookup_plugins/77788.py @@ -0,0 +1,6 @@ +from ansible.plugins.lookup import LookupBase + + +class LookupModule(LookupBase): + def run(self, terms, variables, **kwargs): + return {'one': 1, 'two': 2} diff --git a/test/integration/targets/templating_lookups/template_lookups/tasks/main.yml b/test/integration/targets/templating_lookups/template_lookups/tasks/main.yml index f240a234..430ac917 100644 --- a/test/integration/targets/templating_lookups/template_lookups/tasks/main.yml +++ b/test/integration/targets/templating_lookups/template_lookups/tasks/main.yml @@ -87,4 +87,15 @@ that: - password1 != password2 +# 77788 - KeyError when wantlist=False with dict returned +- name: Test that dicts can be parsed with wantlist false + set_fact: + dict_wantlist_true: "{{ lookup('77788', wantlist=True) }}" + dict_wantlist_false: "{{ lookup('77788', wantlist=False) }}" + +- assert: + that: + - dict_wantlist_true is mapping + - dict_wantlist_false is string + - include_tasks: ./errors.yml diff --git a/test/integration/targets/uri/tasks/main.yml b/test/integration/targets/uri/tasks/main.yml index 1d560e5c..a6ba646d 100644 --- a/test/integration/targets/uri/tasks/main.yml +++ b/test/integration/targets/uri/tasks/main.yml @@ -301,6 +301,26 @@ that: - 'result.allow.split(", ")|sort == ["GET", "HEAD", "OPTIONS"]' +- name: Testing support of https_proxy (with failure expected) + environment: + https_proxy: 'https://localhost:3456' + uri: + url: 'https://httpbin.org/get' + register: result + ignore_errors: true + +- assert: + that: + - result is failed + - result.status == -1 + +- name: Testing use_proxy=no is honored + environment: + https_proxy: 'https://localhost:3456' + uri: + url: 'https://httpbin.org/get' + use_proxy: no + # Ubuntu12.04 doesn't have python-urllib3, this makes handling required dependencies a pain across all variations # We'll use this to just skip 12.04 on those tests. We should be sufficiently covered with other OSes and versions - name: Set fact if running on Ubuntu 12.04 diff --git a/test/integration/targets/yum/tasks/yuminstallroot.yml b/test/integration/targets/yum/tasks/yuminstallroot.yml index f9bee6f9..bb69151a 100644 --- a/test/integration/targets/yum/tasks/yuminstallroot.yml +++ b/test/integration/targets/yum/tasks/yuminstallroot.yml @@ -109,6 +109,25 @@ - "yum_result.rc == 0" - "yum_result.changed" - "rpm_result.rc == 0" + - name: remove python before another test + yum: + name: 'python' + state: absent + installroot: "{{ buildah_mount.stdout }}" + releasever: "{{ buildah_host_releasever.stdout }}" + - name: test yum install of python using releasever with latest + yum: + name: 'python' + state: latest + installroot: "{{ buildah_mount.stdout }}" + releasever: "{{ buildah_host_releasever.stdout }}" + register: yum_result + - name: verify installation of python + assert: + that: + - "yum_result.rc == 0" + - "yum_result.changed" + - "rpm_result.rc == 0" always: - name: remove buildah container command: "buildah rm yum_installroot_releasever_test" diff --git a/test/lib/ansible_test/_data/completion/network.txt b/test/lib/ansible_test/_data/completion/network.txt index 8c6243e9..1d6b0c19 100644 --- a/test/lib/ansible_test/_data/completion/network.txt +++ b/test/lib/ansible_test/_data/completion/network.txt @@ -1,2 +1,2 @@ -ios/csr1000v collection=cisco.ios connection=ansible.netcommon.network_cli provider=aws -vyos/1.1.8 collection=vyos.vyos connection=ansible.netcommon.network_cli provider=aws +ios/csr1000v collection=cisco.ios connection=ansible.netcommon.network_cli provider=aws arch=x86_64 +vyos/1.1.8 collection=vyos.vyos connection=ansible.netcommon.network_cli provider=aws arch=x86_64 diff --git a/test/lib/ansible_test/_data/completion/remote.txt b/test/lib/ansible_test/_data/completion/remote.txt index c7a024fb..c07f2d1d 100644 --- a/test/lib/ansible_test/_data/completion/remote.txt +++ b/test/lib/ansible_test/_data/completion/remote.txt @@ -1,8 +1,11 @@ -freebsd/12.3 python=3.8 python_dir=/usr/local/bin provider=aws -freebsd/13.0 python=3.7,2.7,3.8,3.9 python_dir=/usr/local/bin provider=aws -freebsd python_dir=/usr/local/bin provider=aws -macos/12.0 python=3.10 python_dir=/usr/local/bin provider=parallels -macos python_dir=/usr/local/bin provider=parallels -rhel/7.9 python=2.7 provider=aws -rhel/8.5 python=3.6,3.8,3.9 provider=aws -rhel provider=aws +freebsd/12.3 python=3.8 python_dir=/usr/local/bin provider=aws arch=x86_64 +freebsd/13.0 python=3.7,2.7,3.8,3.9 python_dir=/usr/local/bin provider=aws arch=x86_64 +freebsd python_dir=/usr/local/bin provider=aws arch=x86_64 +macos/12.0 python=3.10 python_dir=/usr/local/bin provider=parallels arch=x86_64 +macos python_dir=/usr/local/bin provider=parallels arch=x86_64 +rhel/7.9 python=2.7 provider=aws arch=x86_64 +rhel/8.5 python=3.6,3.8,3.9 provider=aws arch=x86_64 +rhel/9.0 python=3.9 provider=aws arch=x86_64 +rhel provider=aws arch=x86_64 +ubuntu/22.04 python=3.10 provider=aws arch=x86_64 +ubuntu provider=aws arch=x86_64 diff --git a/test/lib/ansible_test/_data/completion/windows.txt b/test/lib/ansible_test/_data/completion/windows.txt index 280ad97f..767c36cb 100644 --- a/test/lib/ansible_test/_data/completion/windows.txt +++ b/test/lib/ansible_test/_data/completion/windows.txt @@ -1,6 +1,6 @@ -windows/2012 provider=aws -windows/2012-R2 provider=aws -windows/2016 provider=aws -windows/2019 provider=aws -windows/2022 provider=aws -windows provider=aws +windows/2012 provider=aws arch=x86_64 +windows/2012-R2 provider=aws arch=x86_64 +windows/2016 provider=aws arch=x86_64 +windows/2019 provider=aws arch=x86_64 +windows/2022 provider=aws arch=x86_64 +windows provider=aws arch=x86_64 diff --git a/test/lib/ansible_test/_data/pytest/config/default.ini b/test/lib/ansible_test/_data/pytest/config/default.ini new file mode 100644 index 00000000..60575bfe --- /dev/null +++ b/test/lib/ansible_test/_data/pytest/config/default.ini @@ -0,0 +1,4 @@ +[pytest] +xfail_strict = true +# avoid using 'mock_use_standalone_module = true' so package maintainers can avoid packaging 'mock' +junit_family = xunit1 diff --git a/test/lib/ansible_test/_data/pytest.ini b/test/lib/ansible_test/_data/pytest/config/legacy.ini index b2668dc2..b2668dc2 100644 --- a/test/lib/ansible_test/_data/pytest.ini +++ b/test/lib/ansible_test/_data/pytest/config/legacy.ini diff --git a/test/lib/ansible_test/_data/requirements/ansible.txt b/test/lib/ansible_test/_data/requirements/ansible.txt index a732a595..20562c3e 100644 --- a/test/lib/ansible_test/_data/requirements/ansible.txt +++ b/test/lib/ansible_test/_data/requirements/ansible.txt @@ -4,10 +4,12 @@ # packages, not optional ones, and with the widest range of versions that could # be suitable) jinja2 >= 3.0.0 -PyYAML +PyYAML >= 5.1 # PyYAML 5.1 is required for Python 3.8+ support cryptography packaging # NOTE: resolvelib 0.x version bumps should be considered major/breaking # NOTE: and we should update the upper cap with care, at least until 1.0 # NOTE: Ref: https://github.com/sarugaku/resolvelib/issues/69 -resolvelib >= 0.5.3, < 0.6.0 # dependency resolver used by ansible-galaxy +# NOTE: When updating the upper bound, also update the latest version used +# NOTE: in the ansible-galaxy-collection test suite. +resolvelib >= 0.5.3, < 0.9.0 # dependency resolver used by ansible-galaxy diff --git a/test/lib/ansible_test/_internal/__init__.py b/test/lib/ansible_test/_internal/__init__.py index e663c45e..43d10c02 100644 --- a/test/lib/ansible_test/_internal/__init__.py +++ b/test/lib/ansible_test/_internal/__init__.py @@ -57,7 +57,7 @@ def main(cli_args=None): # type: (t.Optional[t.List[str]]) -> None display.truncate = config.truncate display.redact = config.redact display.color = config.color - display.info_stderr = config.info_stderr + display.fd = sys.stderr if config.display_stderr else sys.stdout configure_timeout(config) display.info('RLIMIT_NOFILE: %s' % (CURRENT_RLIMIT_NOFILE,), verbosity=2) @@ -66,7 +66,9 @@ def main(cli_args=None): # type: (t.Optional[t.List[str]]) -> None target_names = None try: - data_context().check_layout() + if config.check_layout: + data_context().check_layout() + args.func(config) except PrimeContainers: pass @@ -82,7 +84,7 @@ def main(cli_args=None): # type: (t.Optional[t.List[str]]) -> None if target_names: for target_name in target_names: - print(target_name) # info goes to stderr, this should be on stdout + print(target_name) # display goes to stderr, this should be on stdout display.review_warnings() config.success = True @@ -90,7 +92,7 @@ def main(cli_args=None): # type: (t.Optional[t.List[str]]) -> None display.warning(u'%s' % ex) sys.exit(0) except ApplicationError as ex: - display.error(u'%s' % ex) + display.fatal(u'%s' % ex) sys.exit(1) except KeyboardInterrupt: sys.exit(2) diff --git a/test/lib/ansible_test/_internal/ansible_util.py b/test/lib/ansible_test/_internal/ansible_util.py index a3582dc8..0fe3db26 100644 --- a/test/lib/ansible_test/_internal/ansible_util.py +++ b/test/lib/ansible_test/_internal/ansible_util.py @@ -22,11 +22,11 @@ from .util import ( ANSIBLE_SOURCE_ROOT, ANSIBLE_TEST_TOOLS_ROOT, get_ansible_version, + raw_command, ) from .util_common import ( create_temp_dir, - run_command, ResultType, intercept_python, get_injector_path, @@ -258,12 +258,12 @@ class CollectionDetailError(ApplicationError): self.reason = reason -def get_collection_detail(args, python): # type: (EnvironmentConfig, PythonConfig) -> CollectionDetail +def get_collection_detail(python): # type: (PythonConfig) -> CollectionDetail """Return collection detail.""" collection = data_context().content.collection directory = os.path.join(collection.root, collection.directory) - stdout = run_command(args, [python.path, os.path.join(ANSIBLE_TEST_TOOLS_ROOT, 'collection_detail.py'), directory], capture=True, always=True)[0] + stdout = raw_command([python.path, os.path.join(ANSIBLE_TEST_TOOLS_ROOT, 'collection_detail.py'), directory], capture=True)[0] result = json.loads(stdout) error = result.get('error') @@ -282,15 +282,15 @@ def run_playbook( args, # type: EnvironmentConfig inventory_path, # type: str playbook, # type: str - run_playbook_vars=None, # type: t.Optional[t.Dict[str, t.Any]] - capture=False, # type: bool + capture, # type: bool + variables=None, # type: t.Optional[t.Dict[str, t.Any]] ): # type: (...) -> None """Run the specified playbook using the given inventory file and playbook variables.""" playbook_path = os.path.join(ANSIBLE_TEST_DATA_ROOT, 'playbooks', playbook) cmd = ['ansible-playbook', '-i', inventory_path, playbook_path] - if run_playbook_vars: - cmd.extend(['-e', json.dumps(run_playbook_vars)]) + if variables: + cmd.extend(['-e', json.dumps(variables)]) if args.verbosity: cmd.append('-%s' % ('v' * args.verbosity)) diff --git a/test/lib/ansible_test/_internal/cli/commands/shell.py b/test/lib/ansible_test/_internal/cli/commands/shell.py index 301ff70e..7d52b39e 100644 --- a/test/lib/ansible_test/_internal/cli/commands/shell.py +++ b/test/lib/ansible_test/_internal/cli/commands/shell.py @@ -39,9 +39,21 @@ def do_shell( shell = parser.add_argument_group(title='shell arguments') shell.add_argument( + 'cmd', + nargs='*', + help='run the specified command', + ) + + shell.add_argument( '--raw', action='store_true', help='direct to shell with no setup', ) + shell.add_argument( + '--export', + metavar='PATH', + help='export inventory instead of opening a shell', + ) + add_environments(parser, completer, ControllerMode.DELEGATED, TargetMode.SHELL) # shell diff --git a/test/lib/ansible_test/_internal/cli/compat.py b/test/lib/ansible_test/_internal/cli/compat.py index dfa7cfa6..0a23c230 100644 --- a/test/lib/ansible_test/_internal/cli/compat.py +++ b/test/lib/ansible_test/_internal/cli/compat.py @@ -115,6 +115,7 @@ class LegacyHostOptions: venv_system_site_packages: t.Optional[bool] = None remote: t.Optional[str] = None remote_provider: t.Optional[str] = None + remote_arch: t.Optional[str] = None docker: t.Optional[str] = None docker_privileged: t.Optional[bool] = None docker_seccomp: t.Optional[str] = None @@ -201,6 +202,9 @@ def convert_legacy_args( '--controller', '--target', '--target-python', + '--target-posix', + '--target-windows', + '--target-network', ] used_old_options = old_options.get_options_used() @@ -371,33 +375,34 @@ def get_legacy_host_config( if remote_config.controller_supported: if controller_python(options.python) or not options.python: - controller = PosixRemoteConfig(name=options.remote, python=native_python(options), provider=options.remote_provider) + controller = PosixRemoteConfig(name=options.remote, python=native_python(options), provider=options.remote_provider, + arch=options.remote_arch) targets = controller_targets(mode, options, controller) else: controller_fallback = f'remote:{options.remote}', f'--remote {options.remote} --python {options.python}', FallbackReason.PYTHON - controller = PosixRemoteConfig(name=options.remote, provider=options.remote_provider) + controller = PosixRemoteConfig(name=options.remote, provider=options.remote_provider, arch=options.remote_arch) targets = controller_targets(mode, options, controller) else: context, reason = f'--remote {options.remote}', FallbackReason.ENVIRONMENT controller = None - targets = [PosixRemoteConfig(name=options.remote, python=native_python(options), provider=options.remote_provider)] + targets = [PosixRemoteConfig(name=options.remote, python=native_python(options), provider=options.remote_provider, arch=options.remote_arch)] elif mode == TargetMode.SHELL and options.remote.startswith('windows/'): if options.python and options.python not in CONTROLLER_PYTHON_VERSIONS: raise ControllerNotSupportedError(f'--python {options.python}') controller = OriginConfig(python=native_python(options)) - targets = [WindowsRemoteConfig(name=options.remote, provider=options.remote_provider)] + targets = [WindowsRemoteConfig(name=options.remote, provider=options.remote_provider, arch=options.remote_arch)] else: if not options.python: raise PythonVersionUnspecifiedError(f'--remote {options.remote}') if controller_python(options.python): - controller = PosixRemoteConfig(name=options.remote, python=native_python(options), provider=options.remote_provider) + controller = PosixRemoteConfig(name=options.remote, python=native_python(options), provider=options.remote_provider, arch=options.remote_arch) targets = controller_targets(mode, options, controller) else: context, reason = f'--remote {options.remote} --python {options.python}', FallbackReason.PYTHON controller = None - targets = [PosixRemoteConfig(name=options.remote, python=native_python(options), provider=options.remote_provider)] + targets = [PosixRemoteConfig(name=options.remote, python=native_python(options), provider=options.remote_provider, arch=options.remote_arch)] if not controller: if docker_available(): @@ -455,12 +460,13 @@ def handle_non_posix_targets( """Return a list of non-POSIX targets if the target mode is non-POSIX.""" if mode == TargetMode.WINDOWS_INTEGRATION: if options.windows: - targets = [WindowsRemoteConfig(name=f'windows/{version}', provider=options.remote_provider) for version in options.windows] + targets = [WindowsRemoteConfig(name=f'windows/{version}', provider=options.remote_provider, arch=options.remote_arch) + for version in options.windows] else: targets = [WindowsInventoryConfig(path=options.inventory)] elif mode == TargetMode.NETWORK_INTEGRATION: if options.platform: - network_targets = [NetworkRemoteConfig(name=platform, provider=options.remote_provider) for platform in options.platform] + network_targets = [NetworkRemoteConfig(name=platform, provider=options.remote_provider, arch=options.remote_arch) for platform in options.platform] for platform, collection in options.platform_collection or []: for entry in network_targets: diff --git a/test/lib/ansible_test/_internal/cli/environments.py b/test/lib/ansible_test/_internal/cli/environments.py index 5709c7c1..e3e759fd 100644 --- a/test/lib/ansible_test/_internal/cli/environments.py +++ b/test/lib/ansible_test/_internal/cli/environments.py @@ -13,6 +13,10 @@ from ..constants import ( SUPPORTED_PYTHON_VERSIONS, ) +from ..util import ( + REMOTE_ARCHITECTURES, +) + from ..completion import ( docker_completion, network_completion, @@ -532,6 +536,13 @@ def add_environment_remote( help=suppress or 'remote provider to use: %(choices)s', ) + environments_parser.add_argument( + '--remote-arch', + metavar='ARCH', + choices=REMOTE_ARCHITECTURES, + help=suppress or 'remote arch to use: %(choices)s', + ) + def complete_remote_stage(prefix: str, **_) -> t.List[str]: """Return a list of supported stages matching the given prefix.""" diff --git a/test/lib/ansible_test/_internal/cli/parsers/key_value_parsers.py b/test/lib/ansible_test/_internal/cli/parsers/key_value_parsers.py index b22705f7..8f71e763 100644 --- a/test/lib/ansible_test/_internal/cli/parsers/key_value_parsers.py +++ b/test/lib/ansible_test/_internal/cli/parsers/key_value_parsers.py @@ -10,6 +10,10 @@ from ...constants import ( SUPPORTED_PYTHON_VERSIONS, ) +from ...util import ( + REMOTE_ARCHITECTURES, +) + from ...host_configs import ( OriginConfig, ) @@ -126,6 +130,7 @@ class PosixRemoteKeyValueParser(KeyValueParser): """Return a dictionary of key names and value parsers.""" return dict( provider=ChoicesParser(REMOTE_PROVIDERS), + arch=ChoicesParser(REMOTE_ARCHITECTURES), python=PythonParser(versions=self.versions, allow_venv=False, allow_default=self.allow_default), ) @@ -137,6 +142,7 @@ class PosixRemoteKeyValueParser(KeyValueParser): state.sections[f'{"controller" if self.controller else "target"} {section_name} (comma separated):'] = '\n'.join([ f' provider={ChoicesParser(REMOTE_PROVIDERS).document(state)}', + f' arch={ChoicesParser(REMOTE_ARCHITECTURES).document(state)}', f' python={python_parser.document(state)}', ]) @@ -149,6 +155,7 @@ class WindowsRemoteKeyValueParser(KeyValueParser): """Return a dictionary of key names and value parsers.""" return dict( provider=ChoicesParser(REMOTE_PROVIDERS), + arch=ChoicesParser(REMOTE_ARCHITECTURES), ) def document(self, state): # type: (DocumentationState) -> t.Optional[str] @@ -157,6 +164,7 @@ class WindowsRemoteKeyValueParser(KeyValueParser): state.sections[f'target {section_name} (comma separated):'] = '\n'.join([ f' provider={ChoicesParser(REMOTE_PROVIDERS).document(state)}', + f' arch={ChoicesParser(REMOTE_ARCHITECTURES).document(state)}', ]) return f'{{{section_name}}}' @@ -168,6 +176,7 @@ class NetworkRemoteKeyValueParser(KeyValueParser): """Return a dictionary of key names and value parsers.""" return dict( provider=ChoicesParser(REMOTE_PROVIDERS), + arch=ChoicesParser(REMOTE_ARCHITECTURES), collection=AnyParser(), connection=AnyParser(), ) @@ -178,7 +187,8 @@ class NetworkRemoteKeyValueParser(KeyValueParser): state.sections[f'target {section_name} (comma separated):'] = '\n'.join([ f' provider={ChoicesParser(REMOTE_PROVIDERS).document(state)}', - ' collection={collecton}', + f' arch={ChoicesParser(REMOTE_ARCHITECTURES).document(state)}', + ' collection={collection}', ' connection={connection}', ]) diff --git a/test/lib/ansible_test/_internal/commands/coverage/__init__.py b/test/lib/ansible_test/_internal/commands/coverage/__init__.py index 1e59ac6f..88128c46 100644 --- a/test/lib/ansible_test/_internal/commands/coverage/__init__.py +++ b/test/lib/ansible_test/_internal/commands/coverage/__init__.py @@ -95,7 +95,16 @@ def run_coverage(args, host_state, output_file, command, cmd): # type: (Coverag cmd = ['python', '-m', 'coverage.__main__', command, '--rcfile', COVERAGE_CONFIG_PATH] + cmd - intercept_python(args, host_state.controller_profile.python, cmd, env) + stdout, stderr = intercept_python(args, host_state.controller_profile.python, cmd, env, capture=True) + + stdout = (stdout or '').strip() + stderr = (stderr or '').strip() + + if stdout: + display.info(stdout) + + if stderr: + display.warning(stderr) def get_all_coverage_files(): # type: () -> t.List[str] diff --git a/test/lib/ansible_test/_internal/commands/coverage/analyze/__init__.py b/test/lib/ansible_test/_internal/commands/coverage/analyze/__init__.py index db169fd7..16521bef 100644 --- a/test/lib/ansible_test/_internal/commands/coverage/analyze/__init__.py +++ b/test/lib/ansible_test/_internal/commands/coverage/analyze/__init__.py @@ -14,4 +14,4 @@ class CoverageAnalyzeConfig(CoverageConfig): # avoid mixing log messages with file output when using `/dev/stdout` for the output file on commands # this may be worth considering as the default behavior in the future, instead of being dependent on the command or options used - self.info_stderr = True + self.display_stderr = True diff --git a/test/lib/ansible_test/_internal/commands/coverage/analyze/targets/__init__.py b/test/lib/ansible_test/_internal/commands/coverage/analyze/targets/__init__.py index f94b7360..26796988 100644 --- a/test/lib/ansible_test/_internal/commands/coverage/analyze/targets/__init__.py +++ b/test/lib/ansible_test/_internal/commands/coverage/analyze/targets/__init__.py @@ -29,10 +29,6 @@ TargetSetIndexes = t.Dict[t.FrozenSet[int], int] class CoverageAnalyzeTargetsConfig(CoverageAnalyzeConfig): """Configuration for the `coverage analyze targets` command.""" - def __init__(self, args): # type: (t.Any) -> None - super().__init__(args) - - self.info_stderr = True def make_report(target_indexes, arcs, lines): # type: (TargetIndexes, Arcs, Lines) -> t.Dict[str, t.Any] diff --git a/test/lib/ansible_test/_internal/commands/coverage/combine.py b/test/lib/ansible_test/_internal/commands/coverage/combine.py index 8cf4c105..8458081f 100644 --- a/test/lib/ansible_test/_internal/commands/coverage/combine.py +++ b/test/lib/ansible_test/_internal/commands/coverage/combine.py @@ -18,11 +18,11 @@ from ...util import ( ANSIBLE_TEST_TOOLS_ROOT, display, ApplicationError, + raw_command, ) from ...util_common import ( ResultType, - run_command, write_json_file, write_json_test_results, ) @@ -194,7 +194,7 @@ def _command_coverage_combine_powershell(args): # type: (CoverageCombineConfig) cmd = ['pwsh', os.path.join(ANSIBLE_TEST_TOOLS_ROOT, 'coverage_stub.ps1')] cmd.extend(source_paths) - stubs = json.loads(run_command(args, cmd, capture=True, always=True)[0]) + stubs = json.loads(raw_command(cmd, capture=True)[0]) return dict((d['Path'], dict((line, 0) for line in d['Lines'])) for d in stubs) diff --git a/test/lib/ansible_test/_internal/commands/integration/__init__.py b/test/lib/ansible_test/_internal/commands/integration/__init__.py index 247bce08..d5f497b5 100644 --- a/test/lib/ansible_test/_internal/commands/integration/__init__.py +++ b/test/lib/ansible_test/_internal/commands/integration/__init__.py @@ -619,7 +619,7 @@ def command_integration_script( cmd += ['-e', '@%s' % config_path] env.update(coverage_manager.get_environment(target.name, target.aliases)) - cover_python(args, host_state.controller_profile.python, cmd, target.name, env, cwd=cwd) + cover_python(args, host_state.controller_profile.python, cmd, target.name, env, cwd=cwd, capture=False) def command_integration_role( @@ -738,7 +738,7 @@ def command_integration_role( env['ANSIBLE_ROLES_PATH'] = test_env.targets_dir env.update(coverage_manager.get_environment(target.name, target.aliases)) - cover_python(args, host_state.controller_profile.python, cmd, target.name, env, cwd=cwd) + cover_python(args, host_state.controller_profile.python, cmd, target.name, env, cwd=cwd, capture=False) def run_setup_targets( diff --git a/test/lib/ansible_test/_internal/commands/integration/cloud/aws.py b/test/lib/ansible_test/_internal/commands/integration/cloud/aws.py index b2b02095..a67a0f89 100644 --- a/test/lib/ansible_test/_internal/commands/integration/cloud/aws.py +++ b/test/lib/ansible_test/_internal/commands/integration/cloud/aws.py @@ -21,6 +21,7 @@ from ....target import ( from ....core_ci import ( AnsibleCoreCI, + CloudResource, ) from ....host_configs import ( @@ -91,7 +92,7 @@ class AwsCloudProvider(CloudProvider): def _create_ansible_core_ci(self): # type: () -> AnsibleCoreCI """Return an AWS instance of AnsibleCoreCI.""" - return AnsibleCoreCI(self.args, 'aws', 'aws', 'aws', persist=False) + return AnsibleCoreCI(self.args, CloudResource(platform='aws')) class AwsCloudEnvironment(CloudEnvironment): diff --git a/test/lib/ansible_test/_internal/commands/integration/cloud/azure.py b/test/lib/ansible_test/_internal/commands/integration/cloud/azure.py index cf16c7f5..f67d1adf 100644 --- a/test/lib/ansible_test/_internal/commands/integration/cloud/azure.py +++ b/test/lib/ansible_test/_internal/commands/integration/cloud/azure.py @@ -19,6 +19,7 @@ from ....target import ( from ....core_ci import ( AnsibleCoreCI, + CloudResource, ) from . import ( @@ -97,7 +98,7 @@ class AzureCloudProvider(CloudProvider): def _create_ansible_core_ci(self): # type: () -> AnsibleCoreCI """Return an Azure instance of AnsibleCoreCI.""" - return AnsibleCoreCI(self.args, 'azure', 'azure', 'azure', persist=False) + return AnsibleCoreCI(self.args, CloudResource(platform='azure')) class AzureCloudEnvironment(CloudEnvironment): diff --git a/test/lib/ansible_test/_internal/commands/integration/cloud/cs.py b/test/lib/ansible_test/_internal/commands/integration/cloud/cs.py index f20a7d88..8ffcabfb 100644 --- a/test/lib/ansible_test/_internal/commands/integration/cloud/cs.py +++ b/test/lib/ansible_test/_internal/commands/integration/cloud/cs.py @@ -106,7 +106,7 @@ class CsCloudProvider(CloudProvider): # apply work-around for OverlayFS issue # https://github.com/docker/for-linux/issues/72#issuecomment-319904698 - docker_exec(self.args, self.DOCKER_SIMULATOR_NAME, ['find', '/var/lib/mysql', '-type', 'f', '-exec', 'touch', '{}', ';']) + docker_exec(self.args, self.DOCKER_SIMULATOR_NAME, ['find', '/var/lib/mysql', '-type', 'f', '-exec', 'touch', '{}', ';'], capture=True) if self.args.explain: values = dict( diff --git a/test/lib/ansible_test/_internal/commands/integration/cloud/hcloud.py b/test/lib/ansible_test/_internal/commands/integration/cloud/hcloud.py index 28b07e72..6912aff3 100644 --- a/test/lib/ansible_test/_internal/commands/integration/cloud/hcloud.py +++ b/test/lib/ansible_test/_internal/commands/integration/cloud/hcloud.py @@ -18,6 +18,7 @@ from ....target import ( from ....core_ci import ( AnsibleCoreCI, + CloudResource, ) from . import ( @@ -78,7 +79,7 @@ class HcloudCloudProvider(CloudProvider): def _create_ansible_core_ci(self): # type: () -> AnsibleCoreCI """Return a Heztner instance of AnsibleCoreCI.""" - return AnsibleCoreCI(self.args, 'hetzner', 'hetzner', 'hetzner', persist=False) + return AnsibleCoreCI(self.args, CloudResource(platform='hetzner')) class HcloudCloudEnvironment(CloudEnvironment): diff --git a/test/lib/ansible_test/_internal/commands/integration/coverage.py b/test/lib/ansible_test/_internal/commands/integration/coverage.py index 6b8a0a6e..3146181a 100644 --- a/test/lib/ansible_test/_internal/commands/integration/coverage.py +++ b/test/lib/ansible_test/_internal/commands/integration/coverage.py @@ -118,7 +118,7 @@ class CoverageHandler(t.Generic[THostConfig], metaclass=abc.ABCMeta): def run_playbook(self, playbook, variables): # type: (str, t.Dict[str, str]) -> None """Run the specified playbook using the current inventory.""" self.create_inventory() - run_playbook(self.args, self.inventory_path, playbook, variables) + run_playbook(self.args, self.inventory_path, playbook, capture=False, variables=variables) class PosixCoverageHandler(CoverageHandler[PosixConfig]): diff --git a/test/lib/ansible_test/_internal/commands/integration/filters.py b/test/lib/ansible_test/_internal/commands/integration/filters.py index 0396ce92..35acae52 100644 --- a/test/lib/ansible_test/_internal/commands/integration/filters.py +++ b/test/lib/ansible_test/_internal/commands/integration/filters.py @@ -10,6 +10,7 @@ from ...config import ( from ...util import ( cache, + detect_architecture, display, get_type_map, ) @@ -223,6 +224,14 @@ class NetworkInventoryTargetFilter(TargetFilter[NetworkInventoryConfig]): class OriginTargetFilter(PosixTargetFilter[OriginConfig]): """Target filter for localhost.""" + def filter_targets(self, targets, exclude): # type: (t.List[IntegrationTarget], t.Set[str]) -> None + """Filter the list of targets, adding any which this host profile cannot support to the provided exclude list.""" + super().filter_targets(targets, exclude) + + arch = detect_architecture(self.config.python.path) + + if arch: + self.skip(f'skip/{arch}', f'which are not supported by {arch}', targets, exclude) @cache @@ -247,10 +256,7 @@ def get_target_filter(args, configs, controller): # type: (IntegrationConfig, t def get_remote_skip_aliases(config): # type: (RemoteConfig) -> t.Dict[str, str] """Return a dictionary of skip aliases and the reason why they apply.""" - if isinstance(config, PosixRemoteConfig): - return get_platform_skip_aliases(config.platform, config.version, config.arch) - - return get_platform_skip_aliases(config.platform, config.version, None) + return get_platform_skip_aliases(config.platform, config.version, config.arch) def get_platform_skip_aliases(platform, version, arch): # type: (str, str, t.Optional[str]) -> t.Dict[str, str] diff --git a/test/lib/ansible_test/_internal/commands/sanity/__init__.py b/test/lib/ansible_test/_internal/commands/sanity/__init__.py index d819c37e..30695110 100644 --- a/test/lib/ansible_test/_internal/commands/sanity/__init__.py +++ b/test/lib/ansible_test/_internal/commands/sanity/__init__.py @@ -162,6 +162,8 @@ def command_sanity(args): # type: (SanityConfig) -> None targets_use_pypi = any(isinstance(test, SanityMultipleVersion) and test.needs_pypi for test in tests) and not args.list_tests host_state = prepare_profiles(args, targets_use_pypi=targets_use_pypi) # sanity + get_content_config(args) # make sure content config has been parsed prior to delegation + if args.delegate: raise Delegate(host_state=host_state, require=changes, exclude=args.exclude) @@ -179,7 +181,7 @@ def command_sanity(args): # type: (SanityConfig) -> None for test in tests: if args.list_tests: - display.info(test.name) + print(test.name) # display goes to stderr, this should be on stdout continue for version in SUPPORTED_PYTHON_VERSIONS: @@ -220,7 +222,7 @@ def command_sanity(args): # type: (SanityConfig) -> None all_targets = SanityTargets.filter_and_inject_targets(test, all_targets) usable_targets = SanityTargets.filter_and_inject_targets(test, usable_targets) - usable_targets = sorted(test.filter_targets_by_version(list(usable_targets), version)) + usable_targets = sorted(test.filter_targets_by_version(args, list(usable_targets), version)) usable_targets = settings.filter_skipped_targets(usable_targets) sanity_targets = SanityTargets(tuple(all_targets), tuple(usable_targets)) @@ -362,12 +364,12 @@ class SanityIgnoreParser: for python_version in test.supported_python_versions: test_name = '%s-%s' % (test.name, python_version) - paths_by_test[test_name] = set(target.path for target in test.filter_targets_by_version(test_targets, python_version)) + paths_by_test[test_name] = set(target.path for target in test.filter_targets_by_version(args, test_targets, python_version)) tests_by_name[test_name] = test else: unversioned_test_names.update(dict(('%s-%s' % (test.name, python_version), test.name) for python_version in SUPPORTED_PYTHON_VERSIONS)) - paths_by_test[test.name] = set(target.path for target in test.filter_targets_by_version(test_targets, '')) + paths_by_test[test.name] = set(target.path for target in test.filter_targets_by_version(args, test_targets, '')) tests_by_name[test.name] = test for line_no, line in enumerate(lines, start=1): @@ -761,7 +763,7 @@ class SanityTest(metaclass=abc.ABCMeta): raise NotImplementedError('Sanity test "%s" must implement "filter_targets" or set "no_targets" to True.' % self.name) - def filter_targets_by_version(self, targets, python_version): # type: (t.List[TestTarget], str) -> t.List[TestTarget] + def filter_targets_by_version(self, args, targets, python_version): # type: (SanityConfig, t.List[TestTarget], str) -> t.List[TestTarget] """Return the given list of test targets, filtered to include only those relevant for the test, taking into account the Python version.""" del python_version # python_version is not used here, but derived classes may make use of it @@ -769,7 +771,7 @@ class SanityTest(metaclass=abc.ABCMeta): if self.py2_compat: # This sanity test is a Python 2.x compatibility test. - content_config = get_content_config() + content_config = get_content_config(args) if content_config.py2_support: # This collection supports Python 2.x. @@ -952,6 +954,7 @@ class SanityCodeSmellTest(SanitySingleVersion): cmd = [python.path, self.path] env = ansible_environment(args, color=False) + env.update(PYTHONUTF8='1') # force all code-smell sanity tests to run with Python UTF-8 Mode enabled pattern = None data = None @@ -1055,15 +1058,15 @@ class SanityMultipleVersion(SanityTest, metaclass=abc.ABCMeta): """A tuple of supported Python versions or None if the test does not depend on specific Python versions.""" return SUPPORTED_PYTHON_VERSIONS - def filter_targets_by_version(self, targets, python_version): # type: (t.List[TestTarget], str) -> t.List[TestTarget] + def filter_targets_by_version(self, args, targets, python_version): # type: (SanityConfig, t.List[TestTarget], str) -> t.List[TestTarget] """Return the given list of test targets, filtered to include only those relevant for the test, taking into account the Python version.""" if not python_version: raise Exception('python_version is required to filter multi-version tests') - targets = super().filter_targets_by_version(targets, python_version) + targets = super().filter_targets_by_version(args, targets, python_version) if python_version in REMOTE_ONLY_PYTHON_VERSIONS: - content_config = get_content_config() + content_config = get_content_config(args) if python_version not in content_config.modules.python_versions: # when a remote-only python version is not supported there are no paths to test diff --git a/test/lib/ansible_test/_internal/commands/sanity/pylint.py b/test/lib/ansible_test/_internal/commands/sanity/pylint.py index 0e6ace8e..370e8998 100644 --- a/test/lib/ansible_test/_internal/commands/sanity/pylint.py +++ b/test/lib/ansible_test/_internal/commands/sanity/pylint.py @@ -141,7 +141,7 @@ class PylintTest(SanitySingleVersion): if data_context().content.collection: try: - collection_detail = get_collection_detail(args, python) + collection_detail = get_collection_detail(python) if not collection_detail.version: display.warning('Skipping pylint collection version checks since no collection version was found.') diff --git a/test/lib/ansible_test/_internal/commands/sanity/validate_modules.py b/test/lib/ansible_test/_internal/commands/sanity/validate_modules.py index e0fbac64..f1d44880 100644 --- a/test/lib/ansible_test/_internal/commands/sanity/validate_modules.py +++ b/test/lib/ansible_test/_internal/commands/sanity/validate_modules.py @@ -121,7 +121,7 @@ class ValidateModulesTest(SanitySingleVersion): cmd.extend(['--collection', data_context().content.collection.directory]) try: - collection_detail = get_collection_detail(args, python) + collection_detail = get_collection_detail(python) if collection_detail.version: cmd.extend(['--collection-version', collection_detail.version]) diff --git a/test/lib/ansible_test/_internal/commands/shell/__init__.py b/test/lib/ansible_test/_internal/commands/shell/__init__.py index 4b205171..e62437ea 100644 --- a/test/lib/ansible_test/_internal/commands/shell/__init__.py +++ b/test/lib/ansible_test/_internal/commands/shell/__init__.py @@ -2,10 +2,12 @@ from __future__ import annotations import os +import sys import typing as t from ...util import ( ApplicationError, + OutputStream, display, ) @@ -38,12 +40,20 @@ from ...host_configs import ( OriginConfig, ) +from ...inventory import ( + create_controller_inventory, + create_posix_inventory, +) + def command_shell(args): # type: (ShellConfig) -> None """Entry point for the `shell` command.""" if args.raw and isinstance(args.targets[0], ControllerConfig): raise ApplicationError('The --raw option has no effect on the controller.') + if not args.export and not args.cmd and not sys.stdin.isatty(): + raise ApplicationError('Standard input must be a TTY to launch a shell.') + host_state = prepare_profiles(args, skip_setup=args.raw) # shell if args.delegate: @@ -57,10 +67,28 @@ def command_shell(args): # type: (ShellConfig) -> None if isinstance(target_profile, ControllerProfile): # run the shell locally unless a target was requested con = LocalConnection(args) # type: Connection + + if args.export: + display.info('Configuring controller inventory.', verbosity=1) + create_controller_inventory(args, args.export, host_state.controller_profile) else: # a target was requested, connect to it over SSH con = target_profile.get_controller_target_connections()[0] + if args.export: + display.info('Configuring target inventory.', verbosity=1) + create_posix_inventory(args, args.export, host_state.target_profiles, True) + + if args.export: + return + + if args.cmd: + # Running a command is assumed to be non-interactive. Only a shell (no command) is interactive. + # If we want to support interactive commands in the future, we'll need an `--interactive` command line option. + # Command stderr output is allowed to mix with our own output, which is all sent to stderr. + con.run(args.cmd, capture=False, interactive=False, output_stream=OutputStream.ORIGINAL) + return + if isinstance(con, SshConnection) and args.raw: cmd = [] # type: t.List[str] elif isinstance(target_profile, PosixProfile): @@ -87,4 +115,4 @@ def command_shell(args): # type: (ShellConfig) -> None else: cmd = [] - con.run(cmd) + con.run(cmd, capture=False, interactive=True) diff --git a/test/lib/ansible_test/_internal/commands/units/__init__.py b/test/lib/ansible_test/_internal/commands/units/__init__.py index 02fae8dd..42330a3b 100644 --- a/test/lib/ansible_test/_internal/commands/units/__init__.py +++ b/test/lib/ansible_test/_internal/commands/units/__init__.py @@ -21,6 +21,7 @@ from ...util import ( ANSIBLE_TEST_DATA_ROOT, display, is_subdir, + str_to_version, SubprocessError, ANSIBLE_LIB_ROOT, ANSIBLE_TEST_TARGET_ROOT, @@ -102,7 +103,7 @@ def command_units(args): # type: (UnitsConfig) -> None paths = [target.path for target in include] - content_config = get_content_config() + content_config = get_content_config(args) supported_remote_python_versions = content_config.modules.python_versions if content_config.modules.controller_only: @@ -235,6 +236,20 @@ def command_units(args): # type: (UnitsConfig) -> None sys.exit() for test_context, python, paths, env in test_sets: + # When using pytest-mock, make sure that features introduced in Python 3.8 are available to older Python versions. + # This is done by enabling the mock_use_standalone_module feature, which forces use of mock even when unittest.mock is available. + # Later Python versions have not introduced additional unittest.mock features, so use of mock is not needed as of Python 3.8. + # If future Python versions introduce new unittest.mock features, they will not be available to older Python versions. + # Having the cutoff at Python 3.8 also eases packaging of ansible-core since no supported controller version requires the use of mock. + # + # NOTE: This only affects use of pytest-mock. + # Collection unit tests may directly import mock, which will be provided by ansible-test when it installs requirements using pip. + # Although mock is available for ansible-core unit tests, they should import units.compat.mock instead. + if str_to_version(python.version) < (3, 8): + config_name = 'legacy.ini' + else: + config_name = 'default.ini' + cmd = [ 'pytest', '--forked', @@ -243,7 +258,7 @@ def command_units(args): # type: (UnitsConfig) -> None '--color', 'yes' if args.color else 'no', '-p', 'no:cacheprovider', - '-c', os.path.join(ANSIBLE_TEST_DATA_ROOT, 'pytest.ini'), + '-c', os.path.join(ANSIBLE_TEST_DATA_ROOT, 'pytest', 'config', config_name), '--junit-xml', os.path.join(ResultType.JUNIT.path, 'python%s-%s-units.xml' % (python.version, test_context)), '--strict-markers', # added in pytest 4.5.0 '--rootdir', data_context().content.root, @@ -275,7 +290,7 @@ def command_units(args): # type: (UnitsConfig) -> None display.info('Unit test %s with Python %s' % (test_context, python.version)) try: - cover_python(args, python, cmd, test_context, env) + cover_python(args, python, cmd, test_context, env, capture=False) except SubprocessError as ex: # pytest exits with status code 5 when all tests are skipped, which isn't an error for our use case if ex.status != 5: diff --git a/test/lib/ansible_test/_internal/completion.py b/test/lib/ansible_test/_internal/completion.py index 7aee99ed..73aee2f2 100644 --- a/test/lib/ansible_test/_internal/completion.py +++ b/test/lib/ansible_test/_internal/completion.py @@ -79,6 +79,7 @@ class PythonCompletionConfig(PosixCompletionConfig, metaclass=abc.ABCMeta): class RemoteCompletionConfig(CompletionConfig): """Base class for completion configuration of remote environments provisioned through Ansible Core CI.""" provider: t.Optional[str] = None + arch: t.Optional[str] = None @property def platform(self): @@ -99,6 +100,9 @@ class RemoteCompletionConfig(CompletionConfig): if not self.provider: raise Exception(f'Remote completion entry "{self.name}" must provide a "provider" setting.') + if not self.arch: + raise Exception(f'Remote completion entry "{self.name}" must provide a "arch" setting.') + @dataclasses.dataclass(frozen=True) class InventoryCompletionConfig(CompletionConfig): @@ -152,6 +156,11 @@ class NetworkRemoteCompletionConfig(RemoteCompletionConfig): """Configuration for remote network platforms.""" collection: str = '' connection: str = '' + placeholder: bool = False + + def __post_init__(self): + if not self.placeholder: + super().__post_init__() @dataclasses.dataclass(frozen=True) @@ -160,6 +169,9 @@ class PosixRemoteCompletionConfig(RemoteCompletionConfig, PythonCompletionConfig placeholder: bool = False def __post_init__(self): + if not self.placeholder: + super().__post_init__() + if not self.supported_pythons: if self.version and not self.placeholder: raise Exception(f'POSIX remote completion entry "{self.name}" must provide a "python" setting.') diff --git a/test/lib/ansible_test/_internal/config.py b/test/lib/ansible_test/_internal/config.py index 0a14a806..64e504cb 100644 --- a/test/lib/ansible_test/_internal/config.py +++ b/test/lib/ansible_test/_internal/config.py @@ -1,6 +1,7 @@ """Configuration classes.""" from __future__ import annotations +import dataclasses import enum import os import sys @@ -48,27 +49,20 @@ class TerminateMode(enum.Enum): return self.name.lower() -class ParsedRemote: - """A parsed version of a "remote" string.""" - def __init__(self, arch, platform, version): # type: (t.Optional[str], str, str) -> None - self.arch = arch - self.platform = platform - self.version = version +@dataclasses.dataclass(frozen=True) +class ModulesConfig: + """Configuration for modules.""" + python_requires: str + python_versions: tuple[str, ...] + controller_only: bool - @staticmethod - def parse(value): # type: (str) -> t.Optional['ParsedRemote'] - """Return a ParsedRemote from the given value or None if the syntax is invalid.""" - parts = value.split('/') - if len(parts) == 2: - arch = None - platform, version = parts - elif len(parts) == 3: - arch, platform, version = parts - else: - return None - - return ParsedRemote(arch, platform, version) +@dataclasses.dataclass(frozen=True) +class ContentConfig: + """Configuration for all content.""" + modules: ModulesConfig + python_versions: tuple[str, ...] + py2_support: bool class EnvironmentConfig(CommonConfig): @@ -82,6 +76,10 @@ class EnvironmentConfig(CommonConfig): self.pypi_proxy = args.pypi_proxy # type: bool self.pypi_endpoint = args.pypi_endpoint # type: t.Optional[str] + # Populated by content_config.get_content_config on the origin. + # Serialized and passed to delegated instances to avoid parsing a second time. + self.content_config = None # type: t.Optional[ContentConfig] + # Set by check_controller_python once HostState has been created by prepare_profiles. # This is here for convenience, to avoid needing to pass HostState to some functions which already have access to EnvironmentConfig. self.controller_python = None # type: t.Optional[PythonConfig] @@ -120,9 +118,11 @@ class EnvironmentConfig(CommonConfig): if config.host_path: settings_path = os.path.join(config.host_path, 'settings.dat') state_path = os.path.join(config.host_path, 'state.dat') + config_path = os.path.join(config.host_path, 'config.dat') files.append((os.path.abspath(settings_path), settings_path)) files.append((os.path.abspath(state_path), state_path)) + files.append((os.path.abspath(config_path), config_path)) data_context().register_payload_callback(host_callback) @@ -237,7 +237,12 @@ class ShellConfig(EnvironmentConfig): def __init__(self, args): # type: (t.Any) -> None super().__init__(args, 'shell') + self.cmd = args.cmd # type: t.List[str] self.raw = args.raw # type: bool + self.check_layout = self.delegate # allow shell to be used without a valid layout as long as no delegation is required + self.interactive = sys.stdin.isatty() and not args.cmd # delegation should only be interactive when stdin is a TTY and no command was given + self.export = args.export # type: t.Optional[str] + self.display_stderr = True class SanityConfig(TestConfig): @@ -253,7 +258,7 @@ class SanityConfig(TestConfig): self.keep_git = args.keep_git # type: bool self.prime_venvs = args.prime_venvs # type: bool - self.info_stderr = self.lint + self.display_stderr = self.lint or self.list_tests if self.keep_git: def git_callback(files): # type: (t.List[t.Tuple[str, str]]) -> None @@ -287,12 +292,12 @@ class IntegrationConfig(TestConfig): self.tags = args.tags self.skip_tags = args.skip_tags self.diff = args.diff - self.no_temp_workdir = args.no_temp_workdir - self.no_temp_unicode = args.no_temp_unicode + self.no_temp_workdir = args.no_temp_workdir # type: bool + self.no_temp_unicode = args.no_temp_unicode # type: bool if self.list_targets: self.explain = True - self.info_stderr = True + self.display_stderr = True def get_ansible_config(self): # type: () -> str """Return the path to the Ansible config for the given config.""" diff --git a/test/lib/ansible_test/_internal/connections.py b/test/lib/ansible_test/_internal/connections.py index 14234b2d..95ad7331 100644 --- a/test/lib/ansible_test/_internal/connections.py +++ b/test/lib/ansible_test/_internal/connections.py @@ -3,7 +3,6 @@ from __future__ import annotations import abc import shlex -import sys import tempfile import typing as t @@ -17,6 +16,7 @@ from .config import ( from .util import ( Display, + OutputStream, SubprocessError, retry, ) @@ -46,10 +46,12 @@ class Connection(metaclass=abc.ABCMeta): @abc.abstractmethod def run(self, command, # type: t.List[str] - capture=False, # type: bool + capture, # type: bool + interactive=False, # type: bool data=None, # type: t.Optional[str] stdin=None, # type: t.Optional[t.IO[bytes]] stdout=None, # type: t.Optional[t.IO[bytes]] + output_stream=None, # type: t.Optional[OutputStream] ): # type: (...) -> t.Tuple[t.Optional[str], t.Optional[str]] """Run the specified command and return the result.""" @@ -60,7 +62,7 @@ class Connection(metaclass=abc.ABCMeta): """Extract the given archive file stream in the specified directory.""" tar_cmd = ['tar', 'oxzf', '-', '-C', chdir] - retry(lambda: self.run(tar_cmd, stdin=src)) + retry(lambda: self.run(tar_cmd, stdin=src, capture=True)) def create_archive(self, chdir, # type: str @@ -82,7 +84,7 @@ class Connection(metaclass=abc.ABCMeta): sh_cmd = ['sh', '-c', ' | '.join(' '.join(shlex.quote(cmd) for cmd in command) for command in commands)] - retry(lambda: self.run(sh_cmd, stdout=dst)) + retry(lambda: self.run(sh_cmd, stdout=dst, capture=True)) class LocalConnection(Connection): @@ -92,10 +94,12 @@ class LocalConnection(Connection): def run(self, command, # type: t.List[str] - capture=False, # type: bool + capture, # type: bool + interactive=False, # type: bool data=None, # type: t.Optional[str] stdin=None, # type: t.Optional[t.IO[bytes]] stdout=None, # type: t.Optional[t.IO[bytes]] + output_stream=None, # type: t.Optional[OutputStream] ): # type: (...) -> t.Tuple[t.Optional[str], t.Optional[str]] """Run the specified command and return the result.""" return run_command( @@ -105,6 +109,8 @@ class LocalConnection(Connection): data=data, stdin=stdin, stdout=stdout, + interactive=interactive, + output_stream=output_stream, ) @@ -130,10 +136,12 @@ class SshConnection(Connection): def run(self, command, # type: t.List[str] - capture=False, # type: bool + capture, # type: bool + interactive=False, # type: bool data=None, # type: t.Optional[str] stdin=None, # type: t.Optional[t.IO[bytes]] stdout=None, # type: t.Optional[t.IO[bytes]] + output_stream=None, # type: t.Optional[OutputStream] ): # type: (...) -> t.Tuple[t.Optional[str], t.Optional[str]] """Run the specified command and return the result.""" options = list(self.options) @@ -143,7 +151,7 @@ class SshConnection(Connection): options.append('-q') - if not data and not stdin and not stdout and sys.stdin.isatty(): + if interactive: options.append('-tt') with tempfile.NamedTemporaryFile(prefix='ansible-test-ssh-debug-', suffix='.log') as ssh_logfile: @@ -166,6 +174,8 @@ class SshConnection(Connection): data=data, stdin=stdin, stdout=stdout, + interactive=interactive, + output_stream=output_stream, error_callback=error_callback, ) @@ -208,10 +218,12 @@ class DockerConnection(Connection): def run(self, command, # type: t.List[str] - capture=False, # type: bool + capture, # type: bool + interactive=False, # type: bool data=None, # type: t.Optional[str] stdin=None, # type: t.Optional[t.IO[bytes]] stdout=None, # type: t.Optional[t.IO[bytes]] + output_stream=None, # type: t.Optional[OutputStream] ): # type: (...) -> t.Tuple[t.Optional[str], t.Optional[str]] """Run the specified command and return the result.""" options = [] @@ -219,7 +231,7 @@ class DockerConnection(Connection): if self.user: options.extend(['--user', self.user]) - if not data and not stdin and not stdout and sys.stdin.isatty(): + if interactive: options.append('-it') return docker_exec( @@ -231,6 +243,8 @@ class DockerConnection(Connection): data=data, stdin=stdin, stdout=stdout, + interactive=interactive, + output_stream=output_stream, ) def inspect(self): # type: () -> DockerInspect diff --git a/test/lib/ansible_test/_internal/containers.py b/test/lib/ansible_test/_internal/containers.py index 5e29c6ac..f4d16714 100644 --- a/test/lib/ansible_test/_internal/containers.py +++ b/test/lib/ansible_test/_internal/containers.py @@ -794,7 +794,7 @@ def forward_ssh_ports( inventory = generate_ssh_inventory(ssh_connections) with named_temporary_file(args, 'ssh-inventory-', '.json', None, inventory) as inventory_path: # type: str - run_playbook(args, inventory_path, playbook, dict(hosts_entries=hosts_entries)) + run_playbook(args, inventory_path, playbook, capture=False, variables=dict(hosts_entries=hosts_entries)) ssh_processes = [] # type: t.List[SshProcess] @@ -827,7 +827,7 @@ def cleanup_ssh_ports( inventory = generate_ssh_inventory(ssh_connections) with named_temporary_file(args, 'ssh-inventory-', '.json', None, inventory) as inventory_path: # type: str - run_playbook(args, inventory_path, playbook, dict(hosts_entries=hosts_entries)) + run_playbook(args, inventory_path, playbook, capture=False, variables=dict(hosts_entries=hosts_entries)) if ssh_processes: for process in ssh_processes: diff --git a/test/lib/ansible_test/_internal/content_config.py b/test/lib/ansible_test/_internal/content_config.py index 10574cc0..39a8d412 100644 --- a/test/lib/ansible_test/_internal/content_config.py +++ b/test/lib/ansible_test/_internal/content_config.py @@ -2,6 +2,7 @@ from __future__ import annotations import os +import pickle import typing as t from .constants import ( @@ -21,6 +22,7 @@ from .compat.yaml import ( ) from .io import ( + open_binary_file, read_text_file, ) @@ -28,54 +30,59 @@ from .util import ( ApplicationError, display, str_to_version, - cache, ) from .data import ( data_context, ) +from .config import ( + EnvironmentConfig, + ContentConfig, + ModulesConfig, +) MISSING = object() -class BaseConfig: - """Base class for content configuration.""" - def __init__(self, data): # type: (t.Any) -> None - if not isinstance(data, dict): - raise Exception('config must be type `dict` not `%s`' % type(data)) - +def parse_modules_config(data: t.Any) -> ModulesConfig: + """Parse the given dictionary as module config and return it.""" + if not isinstance(data, dict): + raise Exception('config must be type `dict` not `%s`' % type(data)) -class ModulesConfig(BaseConfig): - """Configuration for modules.""" - def __init__(self, data): # type: (t.Any) -> None - super().__init__(data) + python_requires = data.get('python_requires', MISSING) - python_requires = data.get('python_requires', MISSING) + if python_requires == MISSING: + raise KeyError('python_requires is required') - if python_requires == MISSING: - raise KeyError('python_requires is required') + return ModulesConfig( + python_requires=python_requires, + python_versions=parse_python_requires(python_requires), + controller_only=python_requires == 'controller', + ) - self.python_requires = python_requires - self.python_versions = parse_python_requires(python_requires) - self.controller_only = python_requires == 'controller' +def parse_content_config(data: t.Any) -> ContentConfig: + """Parse the given dictionary as content config and return it.""" + if not isinstance(data, dict): + raise Exception('config must be type `dict` not `%s`' % type(data)) -class ContentConfig(BaseConfig): - """Configuration for all content.""" - def __init__(self, data): # type: (t.Any) -> None - super().__init__(data) + # Configuration specific to modules/module_utils. + modules = parse_modules_config(data.get('modules', {})) - # Configuration specific to modules/module_utils. - self.modules = ModulesConfig(data.get('modules', {})) + # Python versions supported by the controller, combined with Python versions supported by modules/module_utils. + # Mainly used for display purposes and to limit the Python versions used for sanity tests. + python_versions = tuple(version for version in SUPPORTED_PYTHON_VERSIONS + if version in CONTROLLER_PYTHON_VERSIONS or version in modules.python_versions) - # Python versions supported by the controller, combined with Python versions supported by modules/module_utils. - # Mainly used for display purposes and to limit the Python versions used for sanity tests. - self.python_versions = [version for version in SUPPORTED_PYTHON_VERSIONS - if version in CONTROLLER_PYTHON_VERSIONS or version in self.modules.python_versions] + # True if Python 2.x is supported. + py2_support = any(version for version in python_versions if str_to_version(version)[0] == 2) - # True if Python 2.x is supported. - self.py2_support = any(version for version in self.python_versions if str_to_version(version)[0] == 2) + return ContentConfig( + modules=modules, + python_versions=python_versions, + py2_support=py2_support, + ) def load_config(path): # type: (str) -> t.Optional[ContentConfig] @@ -95,7 +102,7 @@ def load_config(path): # type: (str) -> t.Optional[ContentConfig] return None try: - config = ContentConfig(yaml_value) + config = parse_content_config(yaml_value) except Exception as ex: # pylint: disable=broad-except display.warning('Ignoring config "%s" due a config parsing error: %s' % (path, ex)) return None @@ -105,13 +112,18 @@ def load_config(path): # type: (str) -> t.Optional[ContentConfig] return config -@cache -def get_content_config(): # type: () -> ContentConfig +def get_content_config(args): # type: (EnvironmentConfig) -> ContentConfig """ Parse and return the content configuration (if any) for the current collection. For ansible-core, a default configuration is used. Results are cached. """ + if args.host_path: + args.content_config = deserialize_content_config(os.path.join(args.host_path, 'config.dat')) + + if args.content_config: + return args.content_config + collection_config_path = 'tests/config.yml' config = None @@ -120,7 +132,7 @@ def get_content_config(): # type: () -> ContentConfig config = load_config(collection_config_path) if not config: - config = ContentConfig(dict( + config = parse_content_config(dict( modules=dict( python_requires='default', ), @@ -132,20 +144,36 @@ def get_content_config(): # type: () -> ContentConfig 'This collection provides the Python requirement: %s' % ( ', '.join(SUPPORTED_PYTHON_VERSIONS), config.modules.python_requires)) + args.content_config = config + return config -def parse_python_requires(value): # type: (t.Any) -> t.List[str] +def parse_python_requires(value): # type: (t.Any) -> tuple[str, ...] """Parse the given 'python_requires' version specifier and return the matching Python versions.""" if not isinstance(value, str): raise ValueError('python_requires must must be of type `str` not type `%s`' % type(value)) + versions: tuple[str, ...] + if value == 'default': - versions = list(SUPPORTED_PYTHON_VERSIONS) + versions = SUPPORTED_PYTHON_VERSIONS elif value == 'controller': - versions = list(CONTROLLER_PYTHON_VERSIONS) + versions = CONTROLLER_PYTHON_VERSIONS else: specifier_set = SpecifierSet(value) - versions = [version for version in SUPPORTED_PYTHON_VERSIONS if specifier_set.contains(Version(version))] + versions = tuple(version for version in SUPPORTED_PYTHON_VERSIONS if specifier_set.contains(Version(version))) return versions + + +def serialize_content_config(args: EnvironmentConfig, path: str) -> None: + """Serialize the content config to the given path. If the config has not been loaded, an empty config will be serialized.""" + with open_binary_file(path, 'wb') as config_file: + pickle.dump(args.content_config, config_file) + + +def deserialize_content_config(path: str) -> ContentConfig: + """Deserialize content config from the path.""" + with open_binary_file(path) as config_file: + return pickle.load(config_file) diff --git a/test/lib/ansible_test/_internal/core_ci.py b/test/lib/ansible_test/_internal/core_ci.py index dbb428ae..62d063b2 100644 --- a/test/lib/ansible_test/_internal/core_ci.py +++ b/test/lib/ansible_test/_internal/core_ci.py @@ -1,6 +1,8 @@ """Access Ansible Core CI remote services.""" from __future__ import annotations +import abc +import dataclasses import json import os import re @@ -48,6 +50,65 @@ from .data import ( ) +@dataclasses.dataclass(frozen=True) +class Resource(metaclass=abc.ABCMeta): + """Base class for Ansible Core CI resources.""" + @abc.abstractmethod + def as_tuple(self) -> t.Tuple[str, str, str, str]: + """Return the resource as a tuple of platform, version, architecture and provider.""" + + @abc.abstractmethod + def get_label(self) -> str: + """Return a user-friendly label for this resource.""" + + @property + @abc.abstractmethod + def persist(self) -> bool: + """True if the resource is persistent, otherwise false.""" + + +@dataclasses.dataclass(frozen=True) +class VmResource(Resource): + """Details needed to request a VM from Ansible Core CI.""" + platform: str + version: str + architecture: str + provider: str + tag: str + + def as_tuple(self) -> t.Tuple[str, str, str, str]: + """Return the resource as a tuple of platform, version, architecture and provider.""" + return self.platform, self.version, self.architecture, self.provider + + def get_label(self) -> str: + """Return a user-friendly label for this resource.""" + return f'{self.platform} {self.version} ({self.architecture}) [{self.tag}] @{self.provider}' + + @property + def persist(self) -> bool: + """True if the resource is persistent, otherwise false.""" + return True + + +@dataclasses.dataclass(frozen=True) +class CloudResource(Resource): + """Details needed to request cloud credentials from Ansible Core CI.""" + platform: str + + def as_tuple(self) -> t.Tuple[str, str, str, str]: + """Return the resource as a tuple of platform, version, architecture and provider.""" + return self.platform, '', '', self.platform + + def get_label(self) -> str: + """Return a user-friendly label for this resource.""" + return self.platform + + @property + def persist(self) -> bool: + """True if the resource is persistent, otherwise false.""" + return False + + class AnsibleCoreCI: """Client for Ansible Core CI services.""" DEFAULT_ENDPOINT = 'https://ansible-core-ci.testing.ansible.com' @@ -55,16 +116,12 @@ class AnsibleCoreCI: def __init__( self, args, # type: EnvironmentConfig - platform, # type: str - version, # type: str - provider, # type: str - persist=True, # type: bool + resource, # type: Resource load=True, # type: bool - suffix=None, # type: t.Optional[str] ): # type: (...) -> None self.args = args - self.platform = platform - self.version = version + self.resource = resource + self.platform, self.version, self.arch, self.provider = self.resource.as_tuple() self.stage = args.remote_stage self.client = HttpClient(args) self.connection = None @@ -73,35 +130,33 @@ class AnsibleCoreCI: self.default_endpoint = args.remote_endpoint or self.DEFAULT_ENDPOINT self.retries = 3 self.ci_provider = get_ci_provider() - self.provider = provider - self.name = '%s-%s' % (self.platform, self.version) + self.label = self.resource.get_label() - if suffix: - self.name += '-' + suffix + stripped_label = re.sub('[^A-Za-z0-9_.]+', '-', self.label).strip('-') - self.path = os.path.expanduser('~/.ansible/test/instances/%s-%s-%s' % (self.name, self.provider, self.stage)) + self.name = f"{stripped_label}-{self.stage}" # turn the label into something suitable for use as a filename + + self.path = os.path.expanduser(f'~/.ansible/test/instances/{self.name}') self.ssh_key = SshKey(args) - if persist and load and self._load(): + if self.resource.persist and load and self._load(): try: - display.info('Checking existing %s/%s instance %s.' % (self.platform, self.version, self.instance_id), - verbosity=1) + display.info(f'Checking existing {self.label} instance using: {self._uri}', verbosity=1) self.connection = self.get(always_raise_on=[404]) - display.info('Loaded existing %s/%s from: %s' % (self.platform, self.version, self._uri), verbosity=1) + display.info(f'Loaded existing {self.label} instance.', verbosity=1) except HttpError as ex: if ex.status != 404: raise self._clear() - display.info('Cleared stale %s/%s instance %s.' % (self.platform, self.version, self.instance_id), - verbosity=1) + display.info(f'Cleared stale {self.label} instance.', verbosity=1) self.instance_id = None self.endpoint = None - elif not persist: + elif not self.resource.persist: self.instance_id = None self.endpoint = None self._clear() @@ -126,8 +181,7 @@ class AnsibleCoreCI: def start(self): """Start instance.""" if self.started: - display.info('Skipping started %s/%s instance %s.' % (self.platform, self.version, self.instance_id), - verbosity=1) + display.info(f'Skipping started {self.label} instance.', verbosity=1) return None return self._start(self.ci_provider.prepare_core_ci_auth()) @@ -135,22 +189,19 @@ class AnsibleCoreCI: def stop(self): """Stop instance.""" if not self.started: - display.info('Skipping invalid %s/%s instance %s.' % (self.platform, self.version, self.instance_id), - verbosity=1) + display.info(f'Skipping invalid {self.label} instance.', verbosity=1) return response = self.client.delete(self._uri) if response.status_code == 404: self._clear() - display.info('Cleared invalid %s/%s instance %s.' % (self.platform, self.version, self.instance_id), - verbosity=1) + display.info(f'Cleared invalid {self.label} instance.', verbosity=1) return if response.status_code == 200: self._clear() - display.info('Stopped running %s/%s instance %s.' % (self.platform, self.version, self.instance_id), - verbosity=1) + display.info(f'Stopped running {self.label} instance.', verbosity=1) return raise self._create_http_error(response) @@ -158,8 +209,7 @@ class AnsibleCoreCI: def get(self, tries=3, sleep=15, always_raise_on=None): # type: (int, int, t.Optional[t.List[int]]) -> t.Optional[InstanceConnection] """Get instance connection information.""" if not self.started: - display.info('Skipping invalid %s/%s instance %s.' % (self.platform, self.version, self.instance_id), - verbosity=1) + display.info(f'Skipping invalid {self.label} instance.', verbosity=1) return None if not always_raise_on: @@ -180,7 +230,7 @@ class AnsibleCoreCI: if not tries or response.status_code in always_raise_on: raise error - display.warning('%s. Trying again after %d seconds.' % (error, sleep)) + display.warning(f'{error}. Trying again after {sleep} seconds.') time.sleep(sleep) if self.args.explain: @@ -216,9 +266,7 @@ class AnsibleCoreCI: status = 'running' if self.connection.running else 'starting' - display.info('Status update: %s/%s on instance %s is %s.' % - (self.platform, self.version, self.instance_id, status), - verbosity=1) + display.info(f'The {self.label} instance is {status}.', verbosity=1) return self.connection @@ -229,16 +277,15 @@ class AnsibleCoreCI: return time.sleep(10) - raise ApplicationError('Timeout waiting for %s/%s instance %s.' % - (self.platform, self.version, self.instance_id)) + raise ApplicationError(f'Timeout waiting for {self.label} instance.') @property def _uri(self): - return '%s/%s/%s/%s' % (self.endpoint, self.stage, self.provider, self.instance_id) + return f'{self.endpoint}/{self.stage}/{self.provider}/{self.instance_id}' def _start(self, auth): """Start instance.""" - display.info('Initializing new %s/%s instance %s.' % (self.platform, self.version, self.instance_id), verbosity=1) + display.info(f'Initializing new {self.label} instance using: {self._uri}', verbosity=1) if self.platform == 'windows': winrm_config = read_text_file(os.path.join(ANSIBLE_TEST_TARGET_ROOT, 'setup', 'ConfigureRemotingForAnsible.ps1')) @@ -249,6 +296,7 @@ class AnsibleCoreCI: config=dict( platform=self.platform, version=self.version, + architecture=self.arch, public_key=self.ssh_key.pub_contents, query=False, winrm_config=winrm_config, @@ -266,7 +314,7 @@ class AnsibleCoreCI: self.started = True self._save() - display.info('Started %s/%s from: %s' % (self.platform, self.version, self._uri), verbosity=1) + display.info(f'Started {self.label} instance.', verbosity=1) if self.args.explain: return {} @@ -277,8 +325,6 @@ class AnsibleCoreCI: tries = self.retries sleep = 15 - display.info('Trying endpoint: %s' % self.endpoint, verbosity=1) - while True: tries -= 1 response = self.client.put(self._uri, data=json.dumps(data), headers=headers) @@ -294,7 +340,7 @@ class AnsibleCoreCI: if not tries: raise error - display.warning('%s. Trying again after %d seconds.' % (error, sleep)) + display.warning(f'{error}. Trying again after {sleep} seconds.') time.sleep(sleep) def _clear(self): @@ -345,14 +391,14 @@ class AnsibleCoreCI: def save(self): # type: () -> t.Dict[str, str] """Save instance details and return as a dictionary.""" return dict( - platform_version='%s/%s' % (self.platform, self.version), + label=self.resource.get_label(), instance_id=self.instance_id, endpoint=self.endpoint, ) @staticmethod def _create_http_error(response): # type: (HttpResponse) -> ApplicationError - """Return an exception created from the given HTTP resposne.""" + """Return an exception created from the given HTTP response.""" response_json = response.json() stack_trace = '' @@ -369,7 +415,7 @@ class AnsibleCoreCI: traceback_lines = traceback.format_list(traceback_lines) trace = '\n'.join([x.rstrip() for x in traceback_lines]) - stack_trace = ('\nTraceback (from remote server):\n%s' % trace) + stack_trace = f'\nTraceback (from remote server):\n{trace}' else: message = str(response_json) @@ -379,7 +425,7 @@ class AnsibleCoreCI: class CoreHttpError(HttpError): """HTTP response as an error.""" def __init__(self, status, remote_message, remote_stack_trace): # type: (int, str, str) -> None - super().__init__(status, '%s%s' % (remote_message, remote_stack_trace)) + super().__init__(status, f'{remote_message}{remote_stack_trace}') self.remote_message = remote_message self.remote_stack_trace = remote_stack_trace @@ -388,8 +434,8 @@ class CoreHttpError(HttpError): class SshKey: """Container for SSH key used to connect to remote instances.""" KEY_TYPE = 'rsa' # RSA is used to maintain compatibility with paramiko and EC2 - KEY_NAME = 'id_%s' % KEY_TYPE - PUB_NAME = '%s.pub' % KEY_NAME + KEY_NAME = f'id_{KEY_TYPE}' + PUB_NAME = f'{KEY_NAME}.pub' @mutex def __init__(self, args): # type: (EnvironmentConfig) -> None @@ -469,7 +515,7 @@ class SshKey: make_dirs(os.path.dirname(key)) if not os.path.isfile(key) or not os.path.isfile(pub): - run_command(args, ['ssh-keygen', '-m', 'PEM', '-q', '-t', self.KEY_TYPE, '-N', '', '-f', key]) + run_command(args, ['ssh-keygen', '-m', 'PEM', '-q', '-t', self.KEY_TYPE, '-N', '', '-f', key], capture=True) if args.explain: return key, pub @@ -502,6 +548,6 @@ class InstanceConnection: def __str__(self): if self.password: - return '%s:%s [%s:%s]' % (self.hostname, self.port, self.username, self.password) + return f'{self.hostname}:{self.port} [{self.username}:{self.password}]' - return '%s:%s [%s]' % (self.hostname, self.port, self.username) + return f'{self.hostname}:{self.port} [{self.username}]' diff --git a/test/lib/ansible_test/_internal/coverage_util.py b/test/lib/ansible_test/_internal/coverage_util.py index 5c489a02..c514079a 100644 --- a/test/lib/ansible_test/_internal/coverage_util.py +++ b/test/lib/ansible_test/_internal/coverage_util.py @@ -48,7 +48,7 @@ def cover_python( cmd, # type: t.List[str] target_name, # type: str env, # type: t.Dict[str, str] - capture=False, # type: bool + capture, # type: bool data=None, # type: t.Optional[str] cwd=None, # type: t.Optional[str] ): # type: (...) -> t.Tuple[t.Optional[str], t.Optional[str]] diff --git a/test/lib/ansible_test/_internal/delegation.py b/test/lib/ansible_test/_internal/delegation.py index 6298bf24..247ae353 100644 --- a/test/lib/ansible_test/_internal/delegation.py +++ b/test/lib/ansible_test/_internal/delegation.py @@ -15,7 +15,6 @@ from .config import ( CommonConfig, EnvironmentConfig, IntegrationConfig, - SanityConfig, ShellConfig, TestConfig, UnitsConfig, @@ -28,6 +27,7 @@ from .util import ( ANSIBLE_BIN_PATH, ANSIBLE_LIB_ROOT, ANSIBLE_TEST_ROOT, + OutputStream, ) from .util_common import ( @@ -68,6 +68,10 @@ from .provisioning import ( HostState, ) +from .content_config import ( + serialize_content_config, +) + @contextlib.contextmanager def delegation_context(args, host_state): # type: (EnvironmentConfig, HostState) -> t.Iterator[None] @@ -81,6 +85,7 @@ def delegation_context(args, host_state): # type: (EnvironmentConfig, HostState with tempfile.TemporaryDirectory(prefix='host-', dir=ResultType.TMP.path) as host_dir: args.host_settings.serialize(os.path.join(host_dir, 'settings.dat')) host_state.serialize(os.path.join(host_dir, 'state.dat')) + serialize_content_config(args, os.path.join(host_dir, 'config.dat')) args.host_path = os.path.join(ResultType.TMP.relative_path, os.path.basename(host_dir)) @@ -160,12 +165,13 @@ def delegate_command(args, host_state, exclude, require): # type: (EnvironmentC os.path.join(content_root, ResultType.COVERAGE.relative_path), ] - con.run(['mkdir', '-p'] + writable_dirs) - con.run(['chmod', '777'] + writable_dirs) - con.run(['chmod', '755', working_directory]) - con.run(['chmod', '644', os.path.join(content_root, args.metadata_path)]) - con.run(['useradd', pytest_user, '--create-home']) - con.run(insert_options(command, options + ['--requirements-mode', 'only'])) + con.run(['mkdir', '-p'] + writable_dirs, capture=True) + con.run(['chmod', '777'] + writable_dirs, capture=True) + con.run(['chmod', '755', working_directory], capture=True) + con.run(['chmod', '644', os.path.join(content_root, args.metadata_path)], capture=True) + con.run(['useradd', pytest_user, '--create-home'], capture=True) + + con.run(insert_options(command, options + ['--requirements-mode', 'only']), capture=False) container = con.inspect() networks = container.get_network_names() @@ -191,7 +197,12 @@ def delegate_command(args, host_state, exclude, require): # type: (EnvironmentC success = False try: - con.run(insert_options(command, options)) + # When delegating, preserve the original separate stdout/stderr streams, but only when the following conditions are met: + # 1) Display output is being sent to stderr. This indicates the output on stdout must be kept separate from stderr. + # 2) The delegation is non-interactive. Interactive mode, which generally uses a TTY, is not compatible with intercepting stdout/stderr. + # The downside to having separate streams is that individual lines of output from each are more likely to appear out-of-order. + output_stream = OutputStream.ORIGINAL if args.display_stderr and not args.interactive else None + con.run(insert_options(command, options), capture=False, interactive=args.interactive, output_stream=output_stream) success = True finally: if host_delegation: @@ -247,11 +258,6 @@ def generate_command( require, # type: t.List[str] ): # type: (...) -> t.List[str] """Generate the command necessary to delegate ansible-test.""" - options = { - '--color': 1, - '--docker-no-pull': 0, - } - cmd = [os.path.join(ansible_bin_path, 'ansible-test')] cmd = [python.path] + cmd @@ -287,16 +293,7 @@ def generate_command( cmd = ['/usr/bin/env'] + env_args + cmd - cmd += list(filter_options(args, args.host_settings.filtered_args, options, exclude, require)) - cmd += ['--color', 'yes' if args.color else 'no'] - - if isinstance(args, SanityConfig): - base_branch = args.base_branch or get_ci_provider().get_base_branch() - - if base_branch: - cmd += ['--base-branch', base_branch] - - cmd.extend(['--host-path', args.host_path]) + cmd += list(filter_options(args, args.host_settings.filtered_args, exclude, require)) return cmd @@ -304,66 +301,55 @@ def generate_command( def filter_options( args, # type: EnvironmentConfig argv, # type: t.List[str] - options, # type: t.Dict[str, int] exclude, # type: t.List[str] require, # type: t.List[str] ): # type: (...) -> t.Iterable[str] """Return an iterable that filters out unwanted CLI options and injects new ones as requested.""" - options = options.copy() - - options['--truncate'] = 1 - options['--redact'] = 0 - options['--no-redact'] = 0 + replace: list[tuple[str, int, t.Optional[t.Union[bool, str, list[str]]]]] = [ + ('--docker-no-pull', 0, False), + ('--truncate', 1, str(args.truncate)), + ('--color', 1, 'yes' if args.color else 'no'), + ('--redact', 0, False), + ('--no-redact', 0, not args.redact), + ('--host-path', 1, args.host_path), + ] if isinstance(args, TestConfig): - options.update({ - '--changed': 0, - '--tracked': 0, - '--untracked': 0, - '--ignore-committed': 0, - '--ignore-staged': 0, - '--ignore-unstaged': 0, - '--changed-from': 1, - '--changed-path': 1, - '--metadata': 1, - '--exclude': 1, - '--require': 1, - }) - elif isinstance(args, SanityConfig): - options.update({ - '--base-branch': 1, - }) - - if isinstance(args, IntegrationConfig): - options.update({ - '--no-temp-unicode': 0, - }) - - for arg in filter_args(argv, options): - yield arg + replace.extend([ + ('--changed', 0, False), + ('--tracked', 0, False), + ('--untracked', 0, False), + ('--ignore-committed', 0, False), + ('--ignore-staged', 0, False), + ('--ignore-unstaged', 0, False), + ('--changed-from', 1, False), + ('--changed-path', 1, False), + ('--metadata', 1, args.metadata_path), + ('--exclude', 1, exclude), + ('--require', 1, require), + ('--base-branch', 1, args.base_branch or get_ci_provider().get_base_branch()), + ]) + + pass_through_args: list[str] = [] + + for arg in filter_args(argv, {option: count for option, count, replacement in replace}): + if arg == '--' or pass_through_args: + pass_through_args.append(arg) + continue - for arg in args.delegate_args: yield arg - for target in exclude: - yield '--exclude' - yield target - - for target in require: - yield '--require' - yield target - - if isinstance(args, TestConfig): - if args.metadata_path: - yield '--metadata' - yield args.metadata_path - - yield '--truncate' - yield '%d' % args.truncate + for option, _count, replacement in replace: + if not replacement: + continue - if not args.redact: - yield '--no-redact' + if isinstance(replacement, bool): + yield option + elif isinstance(replacement, str): + yield from [option, replacement] + elif isinstance(replacement, list): + for item in replacement: + yield from [option, item] - if isinstance(args, IntegrationConfig): - if args.no_temp_unicode: - yield '--no-temp-unicode' + yield from args.delegate_args + yield from pass_through_args diff --git a/test/lib/ansible_test/_internal/docker_util.py b/test/lib/ansible_test/_internal/docker_util.py index 12509076..2ed6504b 100644 --- a/test/lib/ansible_test/_internal/docker_util.py +++ b/test/lib/ansible_test/_internal/docker_util.py @@ -20,6 +20,7 @@ from .util import ( find_executable, SubprocessError, cache, + OutputStream, ) from .util_common import ( @@ -268,7 +269,7 @@ def docker_pull(args, image): # type: (EnvironmentConfig, str) -> None for _iteration in range(1, 10): try: - docker_command(args, ['pull', image]) + docker_command(args, ['pull', image], capture=False) return except SubprocessError: display.warning('Failed to pull docker image "%s". Waiting a few seconds before trying again.' % image) @@ -279,7 +280,7 @@ def docker_pull(args, image): # type: (EnvironmentConfig, str) -> None def docker_cp_to(args, container_id, src, dst): # type: (EnvironmentConfig, str, str, str) -> None """Copy a file to the specified container.""" - docker_command(args, ['cp', src, '%s:%s' % (container_id, dst)]) + docker_command(args, ['cp', src, '%s:%s' % (container_id, dst)], capture=True) def docker_run( @@ -510,10 +511,12 @@ def docker_exec( args, # type: EnvironmentConfig container_id, # type: str cmd, # type: t.List[str] + capture, # type: bool options=None, # type: t.Optional[t.List[str]] - capture=False, # type: bool stdin=None, # type: t.Optional[t.IO[bytes]] stdout=None, # type: t.Optional[t.IO[bytes]] + interactive=False, # type: bool + output_stream=None, # type: t.Optional[OutputStream] data=None, # type: t.Optional[str] ): # type: (...) -> t.Tuple[t.Optional[str], t.Optional[str]] """Execute the given command in the specified container.""" @@ -523,7 +526,8 @@ def docker_exec( if data or stdin or stdout: options.append('-i') - return docker_command(args, ['exec'] + options + [container_id] + cmd, capture=capture, stdin=stdin, stdout=stdout, data=data) + return docker_command(args, ['exec'] + options + [container_id] + cmd, capture=capture, stdin=stdin, stdout=stdout, interactive=interactive, + output_stream=output_stream, data=data) def docker_info(args): # type: (CommonConfig) -> t.Dict[str, t.Any] @@ -541,18 +545,23 @@ def docker_version(args): # type: (CommonConfig) -> t.Dict[str, t.Any] def docker_command( args, # type: CommonConfig cmd, # type: t.List[str] - capture=False, # type: bool + capture, # type: bool stdin=None, # type: t.Optional[t.IO[bytes]] stdout=None, # type: t.Optional[t.IO[bytes]] + interactive=False, # type: bool + output_stream=None, # type: t.Optional[OutputStream] always=False, # type: bool data=None, # type: t.Optional[str] ): # type: (...) -> t.Tuple[t.Optional[str], t.Optional[str]] """Run the specified docker command.""" env = docker_environment() command = [require_docker().command] + if command[0] == 'podman' and _get_podman_remote(): command.append('--remote') - return run_command(args, command + cmd, env=env, capture=capture, stdin=stdin, stdout=stdout, always=always, data=data) + + return run_command(args, command + cmd, env=env, capture=capture, stdin=stdin, stdout=stdout, interactive=interactive, always=always, + output_stream=output_stream, data=data) def docker_environment(): # type: () -> t.Dict[str, str] diff --git a/test/lib/ansible_test/_internal/host_configs.py b/test/lib/ansible_test/_internal/host_configs.py index fee741e8..11a45064 100644 --- a/test/lib/ansible_test/_internal/host_configs.py +++ b/test/lib/ansible_test/_internal/host_configs.py @@ -39,6 +39,7 @@ from .util import ( get_available_python_versions, str_to_version, version_to_str, + Architecture, ) @@ -206,6 +207,7 @@ class RemoteConfig(HostConfig, metaclass=abc.ABCMeta): """Base class for remote host configuration.""" name: t.Optional[str] = None provider: t.Optional[str] = None + arch: t.Optional[str] = None @property def platform(self): # type: () -> str @@ -227,6 +229,7 @@ class RemoteConfig(HostConfig, metaclass=abc.ABCMeta): self.provider = None self.provider = self.provider or defaults.provider or 'aws' + self.arch = self.arch or defaults.arch or Architecture.X86_64 @property def is_managed(self): # type: () -> bool @@ -330,8 +333,6 @@ class DockerConfig(ControllerHostConfig, PosixConfig): @dataclasses.dataclass class PosixRemoteConfig(RemoteConfig, ControllerHostConfig, PosixConfig): """Configuration for a POSIX remote host.""" - arch: t.Optional[str] = None - def get_defaults(self, context): # type: (HostContext) -> PosixRemoteCompletionConfig """Return the default settings.""" return filter_completion(remote_completion()).get(self.name) or remote_completion().get(self.platform) or PosixRemoteCompletionConfig( @@ -388,6 +389,7 @@ class NetworkRemoteConfig(RemoteConfig, NetworkConfig): """Return the default settings.""" return filter_completion(network_completion()).get(self.name) or NetworkRemoteCompletionConfig( name=self.name, + placeholder=True, ) def apply_defaults(self, context, defaults): # type: (HostContext, CompletionConfig) -> None diff --git a/test/lib/ansible_test/_internal/host_profiles.py b/test/lib/ansible_test/_internal/host_profiles.py index 9079c7e9..b15d7d35 100644 --- a/test/lib/ansible_test/_internal/host_profiles.py +++ b/test/lib/ansible_test/_internal/host_profiles.py @@ -40,6 +40,7 @@ from .host_configs import ( from .core_ci import ( AnsibleCoreCI, SshKey, + VmResource, ) from .util import ( @@ -50,6 +51,7 @@ from .util import ( get_type_map, sanitize_host_name, sorted_versions, + InternalError, ) from .util_common import ( @@ -148,7 +150,7 @@ class Inventory: inventory_text = inventory_text.strip() if not args.explain: - write_text_file(path, inventory_text) + write_text_file(path, inventory_text + '\n') display.info(f'>>> Inventory\n{inventory_text}', verbosity=3) @@ -295,12 +297,18 @@ class RemoteProfile(SshTargetHostProfile[TRemoteConfig], metaclass=abc.ABCMeta): def create_core_ci(self, load): # type: (bool) -> AnsibleCoreCI """Create and return an AnsibleCoreCI instance.""" + if not self.config.arch: + raise InternalError(f'No arch specified for config: {self.config}') + return AnsibleCoreCI( args=self.args, - platform=self.config.platform, - version=self.config.version, - provider=self.config.provider, - suffix='controller' if self.controller else 'target', + resource=VmResource( + platform=self.config.platform, + version=self.config.version, + architecture=self.config.arch, + provider=self.config.provider, + tag='controller' if self.controller else 'target', + ), load=load, ) @@ -362,7 +370,7 @@ class DockerProfile(ControllerHostProfile[DockerConfig], SshTargetHostProfile[Do setup_sh = bootstrapper.get_script() shell = setup_sh.splitlines()[0][2:] - docker_exec(self.args, self.container_name, [shell], data=setup_sh) + docker_exec(self.args, self.container_name, [shell], data=setup_sh, capture=False) def deprovision(self): # type: () -> None """Deprovision the host after delegation has completed.""" @@ -484,8 +492,9 @@ class NetworkRemoteProfile(RemoteProfile[NetworkRemoteConfig]): for dummy in range(1, 90): try: - intercept_python(self.args, self.args.controller_python, cmd, env) - except SubprocessError: + intercept_python(self.args, self.args.controller_python, cmd, env, capture=True) + except SubprocessError as ex: + display.warning(str(ex)) time.sleep(10) else: return @@ -547,7 +556,7 @@ class PosixRemoteProfile(ControllerHostProfile[PosixRemoteConfig], RemoteProfile shell = setup_sh.splitlines()[0][2:] ssh = self.get_origin_controller_connection() - ssh.run([shell], data=setup_sh) + ssh.run([shell], data=setup_sh, capture=False) def get_ssh_connection(self): # type: () -> SshConnection """Return an SSH connection for accessing the host.""" @@ -570,6 +579,8 @@ class PosixRemoteProfile(ControllerHostProfile[PosixRemoteConfig], RemoteProfile become = Sudo() elif self.config.platform == 'rhel': become = Sudo() + elif self.config.platform == 'ubuntu': + become = Sudo() else: raise NotImplementedError(f'Become support has not been implemented for platform "{self.config.platform}" and user "{settings.user}" is not root.') @@ -717,8 +728,9 @@ class WindowsRemoteProfile(RemoteProfile[WindowsRemoteConfig]): for dummy in range(1, 120): try: - intercept_python(self.args, self.args.controller_python, cmd, env) - except SubprocessError: + intercept_python(self.args, self.args.controller_python, cmd, env, capture=True) + except SubprocessError as ex: + display.warning(str(ex)) time.sleep(10) else: return diff --git a/test/lib/ansible_test/_internal/pypi_proxy.py b/test/lib/ansible_test/_internal/pypi_proxy.py index 51974d26..f0d77815 100644 --- a/test/lib/ansible_test/_internal/pypi_proxy.py +++ b/test/lib/ansible_test/_internal/pypi_proxy.py @@ -126,7 +126,8 @@ def configure_target_pypi_proxy(args, profile, pypi_endpoint, pypi_hostname): # force = 'yes' if profile.config.is_managed else 'no' - run_playbook(args, inventory_path, 'pypi_proxy_prepare.yml', dict(pypi_endpoint=pypi_endpoint, pypi_hostname=pypi_hostname, force=force), capture=True) + run_playbook(args, inventory_path, 'pypi_proxy_prepare.yml', capture=True, variables=dict( + pypi_endpoint=pypi_endpoint, pypi_hostname=pypi_hostname, force=force)) atexit.register(cleanup_pypi_proxy) diff --git a/test/lib/ansible_test/_internal/python_requirements.py b/test/lib/ansible_test/_internal/python_requirements.py index f67f6598..9086c798 100644 --- a/test/lib/ansible_test/_internal/python_requirements.py +++ b/test/lib/ansible_test/_internal/python_requirements.py @@ -261,7 +261,7 @@ def run_pip( if not args.explain: try: - connection.run([python.path], data=script) + connection.run([python.path], data=script, capture=False) except SubprocessError: script = prepare_pip_script([PipVersion()]) diff --git a/test/lib/ansible_test/_internal/test.py b/test/lib/ansible_test/_internal/test.py index 3e149b15..6d2cf2a3 100644 --- a/test/lib/ansible_test/_internal/test.py +++ b/test/lib/ansible_test/_internal/test.py @@ -265,10 +265,10 @@ class TestFailure(TestResult): message = 'The test `%s` failed. See stderr output for details.' % command path = '' message = TestMessage(message, path) - print(message) + print(message) # display goes to stderr, this should be on stdout else: for message in self.messages: - print(message) + print(message) # display goes to stderr, this should be on stdout def write_junit(self, args): # type: (TestConfig) -> None """Write results to a junit XML file.""" diff --git a/test/lib/ansible_test/_internal/util.py b/test/lib/ansible_test/_internal/util.py index 0ad78882..60901ef5 100644 --- a/test/lib/ansible_test/_internal/util.py +++ b/test/lib/ansible_test/_internal/util.py @@ -1,12 +1,16 @@ """Miscellaneous utility functions and classes.""" from __future__ import annotations +import abc import errno +import enum import fcntl import importlib.util import inspect +import json import keyword import os +import platform import pkgutil import random import re @@ -41,6 +45,7 @@ from .io import ( from .thread import ( mutex, + WrappedThread, ) from .constants import ( @@ -96,6 +101,36 @@ MODE_DIRECTORY = MODE_READ | stat.S_IWUSR | stat.S_IXUSR | stat.S_IXGRP | stat.S MODE_DIRECTORY_WRITE = MODE_DIRECTORY | stat.S_IWGRP | stat.S_IWOTH +class OutputStream(enum.Enum): + """The output stream to use when running a subprocess and redirecting/capturing stdout or stderr.""" + + ORIGINAL = enum.auto() + AUTO = enum.auto() + + def get_buffer(self, original: t.BinaryIO) -> t.BinaryIO: + """Return the correct output buffer to use, taking into account the given original buffer.""" + + if self == OutputStream.ORIGINAL: + return original + + if self == OutputStream.AUTO: + return display.fd.buffer + + raise NotImplementedError(str(self)) + + +class Architecture: + """ + Normalized architecture names. + These are the architectures supported by ansible-test, such as when provisioning remote instances. + """ + X86_64 = 'x86_64' + AARCH64 = 'aarch64' + + +REMOTE_ARCHITECTURES = list(value for key, value in Architecture.__dict__.items() if not key.startswith('__')) + + def is_valid_identifier(value: str) -> bool: """Return True if the given value is a valid non-keyword Python identifier, otherwise return False.""" return value.isidentifier() and not keyword.iskeyword(value) @@ -119,6 +154,58 @@ def cache(func): # type: (t.Callable[[], TValue]) -> t.Callable[[], TValue] return wrapper +@mutex +def detect_architecture(python: str) -> t.Optional[str]: + """Detect the architecture of the specified Python and return a normalized version, or None if it cannot be determined.""" + results: t.Dict[str, t.Optional[str]] + + try: + results = detect_architecture.results # type: ignore[attr-defined] + except AttributeError: + results = detect_architecture.results = {} # type: ignore[attr-defined] + + if python in results: + return results[python] + + if python == sys.executable or os.path.realpath(python) == os.path.realpath(sys.executable): + uname = platform.uname() + else: + data = raw_command([python, '-c', 'import json, platform; print(json.dumps(platform.uname()));'], capture=True)[0] + uname = json.loads(data) + + translation = { + 'x86_64': Architecture.X86_64, # Linux, macOS + 'amd64': Architecture.X86_64, # FreeBSD + 'aarch64': Architecture.AARCH64, # Linux, FreeBSD + 'arm64': Architecture.AARCH64, # FreeBSD + } + + candidates = [] + + if len(uname) >= 5: + candidates.append(uname[4]) + + if len(uname) >= 6: + candidates.append(uname[5]) + + candidates = sorted(set(candidates)) + architectures = sorted(set(arch for arch in [translation.get(candidate) for candidate in candidates] if arch)) + + architecture: t.Optional[str] = None + + if not architectures: + display.warning(f'Unable to determine architecture for Python interpreter "{python}" from: {candidates}') + elif len(architectures) == 1: + architecture = architectures[0] + display.info(f'Detected architecture {architecture} for Python interpreter: {python}', verbosity=1) + else: + display.warning(f'Conflicting architectures detected ({architectures}) for Python interpreter "{python}" from: {candidates}') + + results[python] = architecture + + return architecture + + def filter_args(args, filters): # type: (t.List[str], t.Dict[str, int]) -> t.List[str] """Return a filtered version of the given command line arguments.""" remaining = 0 @@ -254,18 +341,46 @@ def get_available_python_versions(): # type: () -> t.Dict[str, str] def raw_command( cmd, # type: t.Iterable[str] - capture=False, # type: bool + capture, # type: bool env=None, # type: t.Optional[t.Dict[str, str]] data=None, # type: t.Optional[str] cwd=None, # type: t.Optional[str] explain=False, # type: bool stdin=None, # type: t.Optional[t.Union[t.IO[bytes], int]] stdout=None, # type: t.Optional[t.Union[t.IO[bytes], int]] + interactive=False, # type: bool + output_stream=None, # type: t.Optional[OutputStream] cmd_verbosity=1, # type: int str_errors='strict', # type: str error_callback=None, # type: t.Optional[t.Callable[[SubprocessError], None]] ): # type: (...) -> t.Tuple[t.Optional[str], t.Optional[str]] """Run the specified command and return stdout and stderr as a tuple.""" + output_stream = output_stream or OutputStream.AUTO + + if capture and interactive: + raise InternalError('Cannot combine capture=True with interactive=True.') + + if data and interactive: + raise InternalError('Cannot combine data with interactive=True.') + + if stdin and interactive: + raise InternalError('Cannot combine stdin with interactive=True.') + + if stdout and interactive: + raise InternalError('Cannot combine stdout with interactive=True.') + + if stdin and data: + raise InternalError('Cannot combine stdin with data.') + + if stdout and not capture: + raise InternalError('Redirection of stdout requires capture=True to avoid redirection of stderr to stdout.') + + if output_stream != OutputStream.AUTO and capture: + raise InternalError(f'Cannot combine {output_stream=} with capture=True.') + + if output_stream != OutputStream.AUTO and interactive: + raise InternalError(f'Cannot combine {output_stream=} with interactive=True.') + if not cwd: cwd = os.getcwd() @@ -276,7 +391,30 @@ def raw_command( escaped_cmd = ' '.join(shlex.quote(c) for c in cmd) - display.info('Run command: %s' % escaped_cmd, verbosity=cmd_verbosity, truncate=True) + if capture: + description = 'Run' + elif interactive: + description = 'Interactive' + else: + description = 'Stream' + + description += ' command' + + with_types = [] + + if data: + with_types.append('data') + + if stdin: + with_types.append('stdin') + + if stdout: + with_types.append('stdout') + + if with_types: + description += f' with {"/".join(with_types)}' + + display.info(f'{description}: {escaped_cmd}', verbosity=cmd_verbosity, truncate=True) display.info('Working directory: %s' % cwd, verbosity=2) program = find_executable(cmd[0], cwd=cwd, path=env['PATH'], required='warning') @@ -294,17 +432,23 @@ def raw_command( if stdin is not None: data = None - communicate = True elif data is not None: stdin = subprocess.PIPE communicate = True - - if stdout: - communicate = True - - if capture: + elif interactive: + pass # allow the subprocess access to our stdin + else: + stdin = subprocess.DEVNULL + + if not interactive: + # When not running interactively, send subprocess stdout/stderr through a pipe. + # This isolates the stdout/stderr of the subprocess from the current process, and also hides the current TTY from it, if any. + # This prevents subprocesses from sharing stdout/stderr with the current process or each other. + # Doing so allows subprocesses to safely make changes to their file handles, such as making them non-blocking (ssh does this). + # This also maintains consistency between local testing and CI systems, which typically do not provide a TTY. + # To maintain output ordering, a single pipe is used for both stdout/stderr when not capturing output unless the output stream is ORIGINAL. stdout = stdout or subprocess.PIPE - stderr = subprocess.PIPE + stderr = subprocess.PIPE if capture or output_stream == OutputStream.ORIGINAL else subprocess.STDOUT communicate = True else: stderr = None @@ -324,7 +468,8 @@ def raw_command( if communicate: data_bytes = to_optional_bytes(data) - stdout_bytes, stderr_bytes = process.communicate(data_bytes) + stdout_bytes, stderr_bytes = communicate_with_process(process, data_bytes, stdout == subprocess.PIPE, stderr == subprocess.PIPE, capture=capture, + output_stream=output_stream) stdout_text = to_optional_text(stdout_bytes, str_errors) or u'' stderr_text = to_optional_text(stderr_bytes, str_errors) or u'' else: @@ -347,6 +492,122 @@ def raw_command( raise SubprocessError(cmd, status, stdout_text, stderr_text, runtime, error_callback) +def communicate_with_process( + process: subprocess.Popen, + stdin: t.Optional[bytes], + stdout: bool, + stderr: bool, + capture: bool, + output_stream: OutputStream, +) -> t.Tuple[bytes, bytes]: + """Communicate with the specified process, handling stdin/stdout/stderr as requested.""" + threads: t.List[WrappedThread] = [] + reader: t.Type[ReaderThread] + + if capture: + reader = CaptureThread + else: + reader = OutputThread + + if stdin is not None: + threads.append(WriterThread(process.stdin, stdin)) + + if stdout: + stdout_reader = reader(process.stdout, output_stream.get_buffer(sys.stdout.buffer)) + threads.append(stdout_reader) + else: + stdout_reader = None + + if stderr: + stderr_reader = reader(process.stderr, output_stream.get_buffer(sys.stderr.buffer)) + threads.append(stderr_reader) + else: + stderr_reader = None + + for thread in threads: + thread.start() + + for thread in threads: + try: + thread.wait_for_result() + except Exception as ex: # pylint: disable=broad-except + display.error(str(ex)) + + if isinstance(stdout_reader, ReaderThread): + stdout_bytes = b''.join(stdout_reader.lines) + else: + stdout_bytes = b'' + + if isinstance(stderr_reader, ReaderThread): + stderr_bytes = b''.join(stderr_reader.lines) + else: + stderr_bytes = b'' + + process.wait() + + return stdout_bytes, stderr_bytes + + +class WriterThread(WrappedThread): + """Thread to write data to stdin of a subprocess.""" + def __init__(self, handle: t.IO[bytes], data: bytes) -> None: + super().__init__(self._run) + + self.handle = handle + self.data = data + + def _run(self) -> None: + """Workload to run on a thread.""" + try: + self.handle.write(self.data) + self.handle.flush() + finally: + self.handle.close() + + +class ReaderThread(WrappedThread, metaclass=abc.ABCMeta): + """Thread to read stdout from a subprocess.""" + def __init__(self, handle: t.IO[bytes], buffer: t.BinaryIO) -> None: + super().__init__(self._run) + + self.handle = handle + self.buffer = buffer + self.lines = [] # type: t.List[bytes] + + @abc.abstractmethod + def _run(self) -> None: + """Workload to run on a thread.""" + + +class CaptureThread(ReaderThread): + """Thread to capture stdout from a subprocess into a buffer.""" + def _run(self) -> None: + """Workload to run on a thread.""" + src = self.handle + dst = self.lines + + try: + for line in src: + dst.append(line) + finally: + src.close() + + +class OutputThread(ReaderThread): + """Thread to pass stdout from a subprocess to stdout.""" + def _run(self) -> None: + """Workload to run on a thread.""" + src = self.handle + dst = self.buffer + + try: + for line in src: + dst.write(line) + dst.flush() + finally: + src.close() + + def common_environment(): """Common environment used for executing all programs.""" env = dict( @@ -516,7 +777,7 @@ class Display: self.color = sys.stdout.isatty() self.warnings = [] self.warnings_unique = set() - self.info_stderr = False + self.fd = sys.stderr # default to stderr until config is initialized to avoid early messages going to stdout self.rows = 0 self.columns = 0 self.truncate = 0 @@ -528,7 +789,7 @@ class Display: def __warning(self, message): # type: (str) -> None """Internal implementation for displaying a warning message.""" - self.print_message('WARNING: %s' % message, color=self.purple, fd=sys.stderr) + self.print_message('WARNING: %s' % message, color=self.purple) def review_warnings(self): # type: () -> None """Review all warnings which previously occurred.""" @@ -556,23 +817,27 @@ class Display: def notice(self, message): # type: (str) -> None """Display a notice level message.""" - self.print_message('NOTICE: %s' % message, color=self.purple, fd=sys.stderr) + self.print_message('NOTICE: %s' % message, color=self.purple) def error(self, message): # type: (str) -> None """Display an error level message.""" - self.print_message('ERROR: %s' % message, color=self.red, fd=sys.stderr) + self.print_message('ERROR: %s' % message, color=self.red) + + def fatal(self, message): # type: (str) -> None + """Display a fatal level message.""" + self.print_message('FATAL: %s' % message, color=self.red, stderr=True) def info(self, message, verbosity=0, truncate=False): # type: (str, int, bool) -> None """Display an info level message.""" if self.verbosity >= verbosity: color = self.verbosity_colors.get(verbosity, self.yellow) - self.print_message(message, color=color, fd=sys.stderr if self.info_stderr else sys.stdout, truncate=truncate) + self.print_message(message, color=color, truncate=truncate) def print_message( # pylint: disable=locally-disabled, invalid-name self, message, # type: str color=None, # type: t.Optional[str] - fd=sys.stdout, # type: t.IO[str] + stderr=False, # type: bool truncate=False, # type: bool ): # type: (...) -> None """Display a message.""" @@ -592,10 +857,18 @@ class Display: message = message.replace(self.clear, color) message = '%s%s%s' % (color, message, self.clear) + fd = sys.stderr if stderr else self.fd + print(message, file=fd) fd.flush() +class InternalError(Exception): + """An unhandled internal error indicating a bug in the code.""" + def __init__(self, message: str) -> None: + super().__init__(f'An internal error has occurred in ansible-test: {message}') + + class ApplicationError(Exception): """General application error.""" @@ -648,12 +921,15 @@ class MissingEnvironmentVariable(ApplicationError): self.name = name -def retry(func, ex_type=SubprocessError, sleep=10, attempts=10): +def retry(func, ex_type=SubprocessError, sleep=10, attempts=10, warn=True): """Retry the specified function on failure.""" for dummy in range(1, attempts): try: return func() - except ex_type: + except ex_type as ex: + if warn: + display.warning(str(ex)) + time.sleep(sleep) return func() diff --git a/test/lib/ansible_test/_internal/util_common.py b/test/lib/ansible_test/_internal/util_common.py index 99d22c2b..b49f5d48 100644 --- a/test/lib/ansible_test/_internal/util_common.py +++ b/test/lib/ansible_test/_internal/util_common.py @@ -27,6 +27,7 @@ from .util import ( MODE_DIRECTORY, MODE_FILE_EXECUTE, MODE_FILE, + OutputStream, PYTHON_PATHS, raw_command, ANSIBLE_TEST_DATA_ROOT, @@ -126,6 +127,8 @@ class CommonConfig: """Configuration common to all commands.""" def __init__(self, args, command): # type: (t.Any, str) -> None self.command = command + self.interactive = False + self.check_layout = True self.success = None # type: t.Optional[bool] self.color = args.color # type: bool @@ -135,7 +138,7 @@ class CommonConfig: self.truncate = args.truncate # type: int self.redact = args.redact # type: bool - self.info_stderr = False # type: bool + self.display_stderr = False # type: bool self.session_name = generate_name() @@ -369,7 +372,7 @@ def intercept_python( python, # type: PythonConfig cmd, # type: t.List[str] env, # type: t.Dict[str, str] - capture=False, # type: bool + capture, # type: bool data=None, # type: t.Optional[str] cwd=None, # type: t.Optional[str] always=False, # type: bool @@ -399,21 +402,23 @@ def intercept_python( def run_command( args, # type: CommonConfig cmd, # type: t.Iterable[str] - capture=False, # type: bool + capture, # type: bool env=None, # type: t.Optional[t.Dict[str, str]] data=None, # type: t.Optional[str] cwd=None, # type: t.Optional[str] always=False, # type: bool stdin=None, # type: t.Optional[t.IO[bytes]] stdout=None, # type: t.Optional[t.IO[bytes]] + interactive=False, # type: bool + output_stream=None, # type: t.Optional[OutputStream] cmd_verbosity=1, # type: int str_errors='strict', # type: str error_callback=None, # type: t.Optional[t.Callable[[SubprocessError], None]] ): # type: (...) -> t.Tuple[t.Optional[str], t.Optional[str]] """Run the specified command and return stdout and stderr as a tuple.""" explain = args.explain and not always - return raw_command(cmd, capture=capture, env=env, data=data, cwd=cwd, explain=explain, stdin=stdin, stdout=stdout, - cmd_verbosity=cmd_verbosity, str_errors=str_errors, error_callback=error_callback) + return raw_command(cmd, capture=capture, env=env, data=data, cwd=cwd, explain=explain, stdin=stdin, stdout=stdout, interactive=interactive, + output_stream=output_stream, cmd_verbosity=cmd_verbosity, str_errors=str_errors, error_callback=error_callback) def yamlcheck(python): diff --git a/test/lib/ansible_test/_internal/venv.py b/test/lib/ansible_test/_internal/venv.py index 64d8d04c..21dded62 100644 --- a/test/lib/ansible_test/_internal/venv.py +++ b/test/lib/ansible_test/_internal/venv.py @@ -20,6 +20,7 @@ from .util import ( remove_tree, ApplicationError, str_to_version, + raw_command, ) from .util_common import ( @@ -92,7 +93,7 @@ def create_virtual_environment(args, # type: EnvironmentConfig # creating a virtual environment using 'venv' when running in a virtual environment created by 'virtualenv' results # in a copy of the original virtual environment instead of creation of a new one # avoid this issue by only using "real" python interpreters to invoke 'venv' - for real_python in iterate_real_pythons(args, python.version): + for real_python in iterate_real_pythons(python.version): if run_venv(args, real_python, system_site_packages, pip, path): display.info('Created Python %s virtual environment using "venv": %s' % (python.version, path), verbosity=1) return True @@ -128,7 +129,7 @@ def create_virtual_environment(args, # type: EnvironmentConfig return False -def iterate_real_pythons(args, version): # type: (EnvironmentConfig, str) -> t.Iterable[str] +def iterate_real_pythons(version): # type: (str) -> t.Iterable[str] """ Iterate through available real python interpreters of the requested version. The current interpreter will be checked and then the path will be searched. @@ -138,7 +139,7 @@ def iterate_real_pythons(args, version): # type: (EnvironmentConfig, str) -> t. if version_info == sys.version_info[:len(version_info)]: current_python = sys.executable - real_prefix = get_python_real_prefix(args, current_python) + real_prefix = get_python_real_prefix(current_python) if real_prefix: current_python = find_python(version, os.path.join(real_prefix, 'bin')) @@ -159,7 +160,7 @@ def iterate_real_pythons(args, version): # type: (EnvironmentConfig, str) -> t. if found_python == current_python: return - real_prefix = get_python_real_prefix(args, found_python) + real_prefix = get_python_real_prefix(found_python) if real_prefix: found_python = find_python(version, os.path.join(real_prefix, 'bin')) @@ -168,12 +169,12 @@ def iterate_real_pythons(args, version): # type: (EnvironmentConfig, str) -> t. yield found_python -def get_python_real_prefix(args, python_path): # type: (EnvironmentConfig, str) -> t.Optional[str] +def get_python_real_prefix(python_path): # type: (str) -> t.Optional[str] """ Return the real prefix of the specified interpreter or None if the interpreter is not a virtual environment created by 'virtualenv'. """ cmd = [python_path, os.path.join(os.path.join(ANSIBLE_TEST_TARGET_TOOLS_ROOT, 'virtualenvcheck.py'))] - check_result = json.loads(run_command(args, cmd, capture=True, always=True)[0]) + check_result = json.loads(raw_command(cmd, capture=True)[0]) real_prefix = check_result['real_prefix'] return real_prefix diff --git a/test/lib/ansible_test/_util/controller/sanity/code-smell/changelog.py b/test/lib/ansible_test/_util/controller/sanity/code-smell/changelog.py index fe2ba5e3..983eaeb4 100644 --- a/test/lib/ansible_test/_util/controller/sanity/code-smell/changelog.py +++ b/test/lib/ansible_test/_util/controller/sanity/code-smell/changelog.py @@ -47,7 +47,11 @@ def main(): env = os.environ.copy() env.update(PYTHONPATH='%s:%s' % (os.path.join(os.path.dirname(__file__), 'changelog'), env['PYTHONPATH'])) - subprocess.call(cmd, env=env) # ignore the return code, rely on the output instead + # ignore the return code, rely on the output instead + process = subprocess.run(cmd, stdin=subprocess.DEVNULL, capture_output=True, text=True, env=env, check=False) + + sys.stdout.write(process.stdout) + sys.stderr.write(process.stderr) if __name__ == '__main__': diff --git a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py index 0bdd9dee..f18477d5 100644 --- a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py +++ b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/main.py @@ -436,14 +436,13 @@ class ModuleValidator(Validator): base_path = self._get_base_branch_module_path() command = ['git', 'show', '%s:%s' % (self.base_branch, base_path or self.path)] - p = subprocess.Popen(command, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = p.communicate() + p = subprocess.run(command, stdin=subprocess.DEVNULL, capture_output=True, check=False) + if int(p.returncode) != 0: return None t = tempfile.NamedTemporaryFile(delete=False) - t.write(stdout) + t.write(p.stdout) t.close() return t.name @@ -2456,11 +2455,12 @@ class GitCache: @staticmethod def _git(args): cmd = ['git'] + args - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = p.communicate() + p = subprocess.run(cmd, stdin=subprocess.DEVNULL, capture_output=True, text=True, check=False) + if p.returncode != 0: - raise GitError(stderr, p.returncode) - return stdout.decode('utf-8').splitlines() + raise GitError(p.stderr, p.returncode) + + return p.stdout.splitlines() class GitError(Exception): diff --git a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/module_args.py b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/module_args.py index ee938142..03a14019 100644 --- a/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/module_args.py +++ b/test/lib/ansible_test/_util/controller/sanity/validate-modules/validate_modules/module_args.py @@ -122,14 +122,12 @@ def get_ps_argument_spec(filename, collection): }) script_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'ps_argspec.ps1') - proc = subprocess.Popen(['pwsh', script_path, util_manifest], stdout=subprocess.PIPE, stderr=subprocess.PIPE, - shell=False) - stdout, stderr = proc.communicate() + proc = subprocess.run(['pwsh', script_path, util_manifest], stdin=subprocess.DEVNULL, capture_output=True, text=True, check=False) if proc.returncode != 0: - raise AnsibleModuleImportError("STDOUT:\n%s\nSTDERR:\n%s" % (stdout.decode('utf-8'), stderr.decode('utf-8'))) + raise AnsibleModuleImportError("STDOUT:\n%s\nSTDERR:\n%s" % (proc.stdout, proc.stderr)) - kwargs = json.loads(stdout) + kwargs = json.loads(proc.stdout) # the validate-modules code expects the options spec to be under the argument_spec key not options as set in PS kwargs['argument_spec'] = kwargs.pop('options', {}) diff --git a/test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py b/test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py index 95209493..930654fc 100755 --- a/test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py +++ b/test/lib/ansible_test/_util/target/cli/ansible_test_cli_stub.py @@ -27,6 +27,9 @@ def main(args=None): raise SystemExit('This version of ansible-test cannot be executed with Python version %s. Supported Python versions are: %s' % ( version_to_str(sys.version_info[:3]), ', '.join(CONTROLLER_PYTHON_VERSIONS))) + if any(not os.get_blocking(handle.fileno()) for handle in (sys.stdin, sys.stdout, sys.stderr)): + raise SystemExit('Standard input, output and error file handles must be blocking to run ansible-test.') + # noinspection PyProtectedMember from ansible_test._internal import main as cli_main diff --git a/test/lib/ansible_test/_util/target/setup/ConfigureRemotingForAnsible.ps1 b/test/lib/ansible_test/_util/target/setup/ConfigureRemotingForAnsible.ps1 index 1fcbaabc..7cc86abd 100644 --- a/test/lib/ansible_test/_util/target/setup/ConfigureRemotingForAnsible.ps1 +++ b/test/lib/ansible_test/_util/target/setup/ConfigureRemotingForAnsible.ps1 @@ -7,6 +7,21 @@ # the necessary changes to allow Ansible to connect, authenticate and # execute PowerShell commands. # +# IMPORTANT: This script uses self-signed certificates and authentication mechanisms +# that are intended for development environments and evaluation purposes only. +# Production environments and deployments that are exposed on the network should +# use CA-signed certificates and secure authentication mechanisms such as Kerberos. +# +# To run this script in Powershell: +# +# [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +# $url = "https://raw.githubusercontent.com/ansible/ansible/devel/examples/scripts/ConfigureRemotingForAnsible.ps1" +# $file = "$env:temp\ConfigureRemotingForAnsible.ps1" +# +# (New-Object -TypeName System.Net.WebClient).DownloadFile($url, $file) +# +# powershell.exe -ExecutionPolicy ByPass -File $file +# # All events are logged to the Windows EventLog, useful for unattended runs. # # Use option -Verbose in order to see the verbose output messages. diff --git a/test/lib/ansible_test/_util/target/setup/bootstrap.sh b/test/lib/ansible_test/_util/target/setup/bootstrap.sh index 3eeac1dd..80ad16c0 100644 --- a/test/lib/ansible_test/_util/target/setup/bootstrap.sh +++ b/test/lib/ansible_test/_util/target/setup/bootstrap.sh @@ -281,6 +281,39 @@ bootstrap_remote_rhel_pinned_pip_packages() pip_install "${pip_packages}" } +bootstrap_remote_ubuntu() +{ + py_pkg_prefix="python3" + + packages=" + gcc + ${py_pkg_prefix}-dev + ${py_pkg_prefix}-pip + ${py_pkg_prefix}-venv + " + + if [ "${controller}" ]; then + # The resolvelib package is not listed here because the available version (0.8.1) is incompatible with ansible. + # Instead, ansible-test will install it using pip. + packages=" + ${packages} + ${py_pkg_prefix}-cryptography + ${py_pkg_prefix}-jinja2 + ${py_pkg_prefix}-packaging + ${py_pkg_prefix}-yaml + " + fi + + while true; do + # shellcheck disable=SC2086 + apt-get update -qq -y && \ + DEBIAN_FRONTEND=noninteractive apt-get install -qq -y --no-install-recommends ${packages} \ + && break + echo "Failed to install packages. Sleeping before trying again..." + sleep 10 + done +} + bootstrap_docker() { # Required for newer mysql-server packages to install/upgrade on Ubuntu 16.04. @@ -299,6 +332,7 @@ bootstrap_remote() "freebsd") bootstrap_remote_freebsd ;; "macos") bootstrap_remote_macos ;; "rhel") bootstrap_remote_rhel ;; + "ubuntu") bootstrap_remote_ubuntu ;; esac done } diff --git a/test/lib/ansible_test/_util/target/setup/requirements.py b/test/lib/ansible_test/_util/target/setup/requirements.py index 0a29429b..ac71f240 100644 --- a/test/lib/ansible_test/_util/target/setup/requirements.py +++ b/test/lib/ansible_test/_util/target/setup/requirements.py @@ -18,6 +18,7 @@ if DESIRED_RLIMIT_NOFILE < CURRENT_RLIMIT_NOFILE: CURRENT_RLIMIT_NOFILE = DESIRED_RLIMIT_NOFILE import base64 +import contextlib import errno import io import json @@ -41,10 +42,10 @@ except ImportError: try: from urllib.request import urlopen except ImportError: - from urllib import urlopen + # noinspection PyCompatibility,PyUnresolvedReferences + from urllib2 import urlopen # pylint: disable=ansible-bad-import-from ENCODING = 'utf-8' -PAYLOAD = b'{payload}' # base-64 encoded JSON payload which will be populated before this script is executed Text = type(u'') @@ -91,7 +92,20 @@ def bootstrap(pip, options): # type: (str, t.Dict[str, t.Any]) -> None log('Downloading pip %s bootstrap script: %s' % (pip_version, url)) make_dirs(os.path.dirname(cache_path)) - download_file(url, temp_path) + + try: + download_file(url, temp_path) + except Exception as ex: + raise ApplicationError((''' +Download failed: %s + +The bootstrap script can be manually downloaded and saved to: %s + +If you're behind a proxy, consider commenting on the following GitHub issue: + +https://github.com/ansible/ansible/issues/77304 +''' % (ex, cache_path)).strip()) + shutil.move(temp_path, cache_path) log('Cached pip %s bootstrap script: %s' % (pip_version, cache_path)) @@ -196,8 +210,8 @@ def devnull(): # type: () -> t.IO[bytes] def download_file(url, path): # type: (str, str) -> None """Download the given URL to the specified file path.""" with open(to_bytes(path), 'wb') as saved_file: - download = urlopen(url) - shutil.copyfileobj(download, saved_file) + with contextlib.closing(urlopen(url)) as download: + shutil.copyfileobj(download, saved_file) class ApplicationError(Exception): @@ -317,5 +331,7 @@ def to_text(value, errors='strict'): # type: (t.AnyStr, str) -> t.Text raise Exception('value is not bytes or text: %s' % type(value)) +PAYLOAD = b'{payload}' # base-64 encoded JSON payload which will be populated before this script is executed + if __name__ == '__main__': main() diff --git a/test/sanity/code-smell/docs-build.py b/test/sanity/code-smell/docs-build.py index 9461620a..aaa69378 100644 --- a/test/sanity/code-smell/docs-build.py +++ b/test/sanity/code-smell/docs-build.py @@ -29,13 +29,12 @@ def main(): try: cmd = ['make', 'core_singlehtmldocs'] - sphinx = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=docs_dir) - stdout, stderr = sphinx.communicate() + sphinx = subprocess.run(cmd, stdin=subprocess.DEVNULL, capture_output=True, cwd=docs_dir, check=False, text=True) finally: shutil.move(tmp, requirements_txt) - stdout = stdout.decode('utf-8') - stderr = stderr.decode('utf-8') + stdout = sphinx.stdout + stderr = sphinx.stderr if sphinx.returncode != 0: sys.stderr.write("Command '%s' failed with status code: %d\n" % (' '.join(cmd), sphinx.returncode)) diff --git a/test/sanity/code-smell/docs-build.requirements.in b/test/sanity/code-smell/docs-build.requirements.in index f4f8c9b0..797ca326 100644 --- a/test/sanity/code-smell/docs-build.requirements.in +++ b/test/sanity/code-smell/docs-build.requirements.in @@ -1,6 +1,6 @@ jinja2 pyyaml -resolvelib < 0.6.0 +resolvelib < 0.9.0 sphinx == 4.2.0 sphinx-notfound-page sphinx-ansible-theme diff --git a/test/sanity/code-smell/package-data.py b/test/sanity/code-smell/package-data.py index 8e777b48..5497b7ca 100644 --- a/test/sanity/code-smell/package-data.py +++ b/test/sanity/code-smell/package-data.py @@ -61,6 +61,7 @@ def assemble_files_to_ship(complete_file_list): # Generated as part of a build step 'docs/docsite/rst/conf.py', 'docs/docsite/rst/index.rst', + 'docs/docsite/rst/dev_guide/index.rst', # Possibly should be included 'examples/scripts/uptime.py', 'examples/scripts/my_test.py', @@ -171,14 +172,15 @@ def clean_repository(file_list): def create_sdist(tmp_dir): """Create an sdist in the repository""" - create = subprocess.Popen( + create = subprocess.run( ['make', 'snapshot', 'SDIST_DIR=%s' % tmp_dir], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + check=False, ) - stderr = create.communicate()[1] + stderr = create.stderr if create.returncode != 0: raise Exception('make snapshot failed:\n%s' % stderr) @@ -219,15 +221,16 @@ def extract_sdist(sdist_path, tmp_dir): def install_sdist(tmp_dir, sdist_dir): """Install the extracted sdist into the temporary directory""" - install = subprocess.Popen( + install = subprocess.run( ['python', 'setup.py', 'install', '--root=%s' % tmp_dir], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, cwd=os.path.join(tmp_dir, sdist_dir), + check=False, ) - stdout, stderr = install.communicate() + stdout, stderr = install.stdout, install.stderr if install.returncode != 0: raise Exception('sdist install failed:\n%s' % stderr) diff --git a/test/sanity/code-smell/package-data.requirements.in b/test/sanity/code-smell/package-data.requirements.in index 68c2248e..fd362d0e 100644 --- a/test/sanity/code-smell/package-data.requirements.in +++ b/test/sanity/code-smell/package-data.requirements.in @@ -1,7 +1,7 @@ docutils < 0.18 # match version required by sphinx in the docs-build sanity test jinja2 pyyaml # ansible-core requirement -resolvelib < 0.6.0 +resolvelib < 0.9.0 rstcheck straight.plugin antsibull-changelog diff --git a/test/sanity/ignore.txt b/test/sanity/ignore.txt index 00547573..0d2bb352 100644 --- a/test/sanity/ignore.txt +++ b/test/sanity/ignore.txt @@ -4,42 +4,6 @@ docs/docsite/rst/locales/ja/LC_MESSAGES/dev_guide.po no-smart-quotes # Translat examples/scripts/ConfigureRemotingForAnsible.ps1 pslint:PSCustomUseLiteralPath examples/scripts/upgrade_to_ps3.ps1 pslint:PSCustomUseLiteralPath examples/scripts/upgrade_to_ps3.ps1 pslint:PSUseApprovedVerbs -lib/ansible/cli/galaxy.py import-3.8 # unguarded indirect resolvelib import -lib/ansible/galaxy/collection/__init__.py import-3.8 # unguarded resolvelib import -lib/ansible/galaxy/collection/concrete_artifact_manager.py import-3.8 # unguarded resolvelib import -lib/ansible/galaxy/collection/galaxy_api_proxy.py import-3.8 # unguarded resolvelib imports -lib/ansible/galaxy/collection/gpg.py import-3.8 # unguarded resolvelib imports -lib/ansible/galaxy/dependency_resolution/__init__.py import-3.8 # circular imports -lib/ansible/galaxy/dependency_resolution/dataclasses.py import-3.8 # circular imports -lib/ansible/galaxy/dependency_resolution/errors.py import-3.8 # circular imports -lib/ansible/galaxy/dependency_resolution/providers.py import-3.8 # circular imports -lib/ansible/galaxy/dependency_resolution/reporters.py import-3.8 # circular imports -lib/ansible/galaxy/dependency_resolution/resolvers.py import-3.8 # circular imports -lib/ansible/galaxy/dependency_resolution/versioning.py import-3.8 # circular imports -lib/ansible/cli/galaxy.py import-3.9 # unguarded indirect resolvelib import -lib/ansible/galaxy/collection/__init__.py import-3.9 # unguarded resolvelib import -lib/ansible/galaxy/collection/concrete_artifact_manager.py import-3.9 # unguarded resolvelib import -lib/ansible/galaxy/collection/galaxy_api_proxy.py import-3.9 # unguarded resolvelib imports -lib/ansible/galaxy/collection/gpg.py import-3.9 # unguarded resolvelib imports -lib/ansible/galaxy/dependency_resolution/__init__.py import-3.9 # circular imports -lib/ansible/galaxy/dependency_resolution/dataclasses.py import-3.9 # circular imports -lib/ansible/galaxy/dependency_resolution/errors.py import-3.9 # circular imports -lib/ansible/galaxy/dependency_resolution/providers.py import-3.9 # circular imports -lib/ansible/galaxy/dependency_resolution/reporters.py import-3.9 # circular imports -lib/ansible/galaxy/dependency_resolution/resolvers.py import-3.9 # circular imports -lib/ansible/galaxy/dependency_resolution/versioning.py import-3.9 # circular imports -lib/ansible/cli/galaxy.py import-3.10 # unguarded indirect resolvelib import -lib/ansible/galaxy/collection/__init__.py import-3.10 # unguarded resolvelib import -lib/ansible/galaxy/collection/concrete_artifact_manager.py import-3.10 # unguarded resolvelib import -lib/ansible/galaxy/collection/galaxy_api_proxy.py import-3.10 # unguarded resolvelib imports -lib/ansible/galaxy/collection/gpg.py import-3.10 # unguarded resolvelib imports -lib/ansible/galaxy/dependency_resolution/__init__.py import-3.10 # circular imports -lib/ansible/galaxy/dependency_resolution/dataclasses.py import-3.10 # circular imports -lib/ansible/galaxy/dependency_resolution/errors.py import-3.10 # circular imports -lib/ansible/galaxy/dependency_resolution/providers.py import-3.10 # circular imports -lib/ansible/galaxy/dependency_resolution/reporters.py import-3.10 # circular imports -lib/ansible/galaxy/dependency_resolution/resolvers.py import-3.10 # circular imports -lib/ansible/galaxy/dependency_resolution/versioning.py import-3.10 # circular imports lib/ansible/cli/scripts/ansible_connection_cli_stub.py shebang lib/ansible/config/base.yml no-unwanted-files lib/ansible/executor/playbook_executor.py pylint:disallowed-name @@ -167,6 +131,7 @@ test/integration/targets/ansible-test/ansible_collections/ns/col/tests/unit/plug test/integration/targets/ansible-test-docker/ansible_collections/ns/col/plugins/modules/hello.py pylint:relative-beyond-top-level test/integration/targets/ansible-test-docker/ansible_collections/ns/col/tests/unit/plugins/modules/test_hello.py pylint:relative-beyond-top-level test/integration/targets/ansible-test-docker/ansible_collections/ns/col/tests/unit/plugins/module_utils/test_my_util.py pylint:relative-beyond-top-level +test/integration/targets/ansible-test-no-tty/ansible_collections/ns/col/vendored_pty.py pep8!skip # vendored code test/integration/targets/collections_relative_imports/collection_root/ansible_collections/my_ns/my_col/plugins/modules/my_module.py pylint:relative-beyond-top-level test/integration/targets/collections_relative_imports/collection_root/ansible_collections/my_ns/my_col/plugins/module_utils/my_util2.py pylint:relative-beyond-top-level test/integration/targets/collections_relative_imports/collection_root/ansible_collections/my_ns/my_col/plugins/module_utils/my_util3.py pylint:relative-beyond-top-level diff --git a/test/units/_vendor/test_vendor.py b/test/units/_vendor/test_vendor.py index cda0279d..84b850e2 100644 --- a/test/units/_vendor/test_vendor.py +++ b/test/units/_vendor/test_vendor.py @@ -9,7 +9,7 @@ import pkgutil import pytest import sys -from mock import MagicMock, NonCallableMagicMock, patch +from unittest.mock import MagicMock, NonCallableMagicMock, patch def reset_internal_vendor_package(): diff --git a/test/units/ansible_test/ci/util.py b/test/units/ansible_test/ci/util.py index ba8e358b..2273f0ab 100644 --- a/test/units/ansible_test/ci/util.py +++ b/test/units/ansible_test/ci/util.py @@ -44,10 +44,8 @@ def verify_signature(request, public_key_pem): public_key = load_pem_public_key(public_key_pem.encode(), default_backend()) - verifier = public_key.verifier( + public_key.verify( base64.b64decode(signature.encode()), + payload_bytes, ec.ECDSA(hashes.SHA256()), ) - - verifier.update(payload_bytes) - verifier.verify() diff --git a/test/units/cli/test_cli.py b/test/units/cli/test_cli.py index 26285955..79c2b8fb 100644 --- a/test/units/cli/test_cli.py +++ b/test/units/cli/test_cli.py @@ -20,7 +20,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type from units.compat import unittest -from mock import patch, MagicMock +from unittest.mock import patch, MagicMock from units.mock.loader import DictDataLoader diff --git a/test/units/cli/test_console.py b/test/units/cli/test_console.py index fb477bf3..4fc05dd3 100644 --- a/test/units/cli/test_console.py +++ b/test/units/cli/test_console.py @@ -20,7 +20,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type from units.compat import unittest -from mock import patch +from unittest.mock import patch from ansible.cli.console import ConsoleCLI diff --git a/test/units/cli/test_doc.py b/test/units/cli/test_doc.py index 576bdb28..b10f0888 100644 --- a/test/units/cli/test_doc.py +++ b/test/units/cli/test_doc.py @@ -29,9 +29,9 @@ TTY_IFY_DATA = { 'L(the user guide, https://docs.ansible.com/)': 'the user guide <https://docs.ansible.com/>', 'R(the user guide, user-guide)': 'the user guide', # de-rsty refs and anchors - 'yolo :ref:`my boy` does stuff': 'yolo website for `my boy` does stuff', - '.. seealso:: Something amazing': 'See website for: Something amazing', - '.. seealso:: Troublesome multiline\n Stuff goes htere': 'See website for: Troublesome multiline\n Stuff goes htere', + 'yolo :ref:`my boy` does stuff': 'yolo `my boy` does stuff', + '.. seealso:: Something amazing': 'See also: Something amazing', + '.. seealso:: Troublesome multiline\n Stuff goes htere': 'See also: Troublesome multiline\n Stuff goes htere', '.. note:: boring stuff': 'Note: boring stuff', } diff --git a/test/units/cli/test_galaxy.py b/test/units/cli/test_galaxy.py index 1a6bfe04..faaf64de 100644 --- a/test/units/cli/test_galaxy.py +++ b/test/units/cli/test_galaxy.py @@ -41,7 +41,7 @@ from ansible.module_utils._text import to_bytes, to_native, to_text from ansible.utils import context_objects as co from ansible.utils.display import Display from units.compat import unittest -from mock import patch, MagicMock +from unittest.mock import patch, MagicMock @pytest.fixture(autouse='function') @@ -631,7 +631,7 @@ def test_invalid_collection_name_install(name, expected, tmp_path_factory): # Used to be: expected = "Invalid collection name '%s', name must be in the format <namespace>.<collection>" % expected expected = "Neither the collection requirement entry key 'name', nor 'source' point to a concrete resolvable collection artifact. " expected += r"Also 'name' is not an FQCN\. A valid collection name must be in the format <namespace>\.<collection>\. " - expected += r"Please make sure that the namespace and the collection name contain characters from \[a\-zA\-Z0\-9_\] only\." + expected += r"Please make sure that the namespace and the collection name contain characters from \[a\-zA\-Z0\-9_\] only\." gc = GalaxyCLI(args=['ansible-galaxy', 'collection', 'install', name, '-p', os.path.join(install_path, 'install')]) with pytest.raises(AnsibleError, match=expected): @@ -1093,7 +1093,7 @@ def test_parse_requirements_without_mandatory_name_key(requirements_cli, require expected = "Neither the collection requirement entry key 'name', nor 'source' point to a concrete resolvable collection artifact. " expected += r"Also 'name' is not an FQCN\. A valid collection name must be in the format <namespace>\.<collection>\. " - expected += r"Please make sure that the namespace and the collection name contain characters from \[a\-zA\-Z0\-9_\] only\." + expected += r"Please make sure that the namespace and the collection name contain characters from \[a\-zA\-Z0\-9_\] only\." with pytest.raises(AnsibleError, match=expected): requirements_cli._parse_requirements_file(requirements_file) diff --git a/test/units/cli/test_vault.py b/test/units/cli/test_vault.py index 76ffba2f..2304f4d5 100644 --- a/test/units/cli/test_vault.py +++ b/test/units/cli/test_vault.py @@ -24,7 +24,7 @@ import os import pytest from units.compat import unittest -from mock import patch, MagicMock +from unittest.mock import patch, MagicMock from units.mock.vault_helper import TextVaultSecret from ansible import context, errors diff --git a/test/units/compat/mock.py b/test/units/compat/mock.py new file mode 100644 index 00000000..58dc78e0 --- /dev/null +++ b/test/units/compat/mock.py @@ -0,0 +1,23 @@ +""" +Compatibility shim for mock imports in modules and module_utils. +This can be removed once support for Python 2.7 is dropped. +""" +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +try: + from unittest.mock import ( + call, + patch, + mock_open, + MagicMock, + Mock, + ) +except ImportError: + from mock import ( + call, + patch, + mock_open, + MagicMock, + Mock, + ) diff --git a/test/units/errors/test_errors.py b/test/units/errors/test_errors.py index deb3dc0b..005be29e 100644 --- a/test/units/errors/test_errors.py +++ b/test/units/errors/test_errors.py @@ -21,7 +21,7 @@ __metaclass__ = type from units.compat import unittest -from mock import mock_open, patch +from unittest.mock import mock_open, patch from ansible.errors import AnsibleError from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject diff --git a/test/units/executor/test_interpreter_discovery.py b/test/units/executor/test_interpreter_discovery.py index 5efdd378..43db5950 100644 --- a/test/units/executor/test_interpreter_discovery.py +++ b/test/units/executor/test_interpreter_discovery.py @@ -6,7 +6,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -from mock import MagicMock +from unittest.mock import MagicMock from ansible.executor.interpreter_discovery import discover_interpreter from ansible.module_utils._text import to_text diff --git a/test/units/executor/test_play_iterator.py b/test/units/executor/test_play_iterator.py index 3ced9e3c..14c28135 100644 --- a/test/units/executor/test_play_iterator.py +++ b/test/units/executor/test_play_iterator.py @@ -20,7 +20,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type from units.compat import unittest -from mock import patch, MagicMock +from unittest.mock import patch, MagicMock from ansible.executor.play_iterator import HostState, PlayIterator, IteratingStates, FailedStates from ansible.playbook import Playbook diff --git a/test/units/executor/test_playbook_executor.py b/test/units/executor/test_playbook_executor.py index 350f7c2d..6032dbb2 100644 --- a/test/units/executor/test_playbook_executor.py +++ b/test/units/executor/test_playbook_executor.py @@ -20,7 +20,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type from units.compat import unittest -from mock import MagicMock +from unittest.mock import MagicMock from ansible.executor.playbook_executor import PlaybookExecutor from ansible.playbook import Playbook diff --git a/test/units/executor/test_task_executor.py b/test/units/executor/test_task_executor.py index 30d609a1..003cedee 100644 --- a/test/units/executor/test_task_executor.py +++ b/test/units/executor/test_task_executor.py @@ -19,20 +19,24 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -import mock +from unittest import mock from units.compat import unittest -from mock import patch, MagicMock +from unittest.mock import patch, MagicMock from ansible.errors import AnsibleError from ansible.executor.task_executor import TaskExecutor, remove_omit -from ansible.plugins.loader import action_loader, lookup_loader +from ansible.plugins.loader import action_loader, lookup_loader, module_loader from ansible.parsing.yaml.objects import AnsibleUnicode from ansible.utils.unsafe_proxy import AnsibleUnsafeText, AnsibleUnsafeBytes from ansible.module_utils.six import text_type +from collections import namedtuple from units.mock.loader import DictDataLoader +get_with_context_result = namedtuple('get_with_context_result', ['object', 'plugin_load_context']) + + class TestTaskExecutor(unittest.TestCase): def test_task_executor_init(self): @@ -204,6 +208,8 @@ class TestTaskExecutor(unittest.TestCase): final_q=MagicMock(), ) + context = MagicMock(resolved=False) + te._shared_loader_obj.module_loader.find_plugin_with_context.return_value = context action_loader = te._shared_loader_obj.action_loader action_loader.has_plugin.return_value = True action_loader.get.return_value = mock.sentinel.handler @@ -238,6 +244,8 @@ class TestTaskExecutor(unittest.TestCase): final_q=MagicMock(), ) + context = MagicMock(resolved=False) + te._shared_loader_obj.module_loader.find_plugin_with_context.return_value = context action_loader = te._shared_loader_obj.action_loader action_loader.has_plugin.side_effect = [False, True] action_loader.get.return_value = mock.sentinel.handler @@ -252,7 +260,7 @@ class TestTaskExecutor(unittest.TestCase): handler = te._get_action_handler(mock_connection, mock_templar) self.assertIs(mock.sentinel.handler, handler) - action_loader.has_plugin.assert_has_calls([mock.call(action, collection_list=te._task.collections), + action_loader.has_plugin.assert_has_calls([mock.call(action, collection_list=te._task.collections), # called twice mock.call(module_prefix, collection_list=te._task.collections)]) action_loader.get.assert_called_once_with( @@ -277,6 +285,9 @@ class TestTaskExecutor(unittest.TestCase): action_loader.has_plugin.return_value = False action_loader.get.return_value = mock.sentinel.handler action_loader.__contains__.return_value = False + module_loader = te._shared_loader_obj.module_loader + context = MagicMock(resolved=False) + module_loader.find_plugin_with_context.return_value = context mock_connection = MagicMock() mock_templar = MagicMock() @@ -302,6 +313,7 @@ class TestTaskExecutor(unittest.TestCase): mock_host = MagicMock() mock_task = MagicMock() + mock_task.action = 'mock.action' mock_task.args = dict() mock_task.retries = 0 mock_task.delay = -1 @@ -328,7 +340,7 @@ class TestTaskExecutor(unittest.TestCase): mock_action = MagicMock() mock_queue = MagicMock() - shared_loader = None + shared_loader = MagicMock() new_stdin = None job_vars = dict(omit="XXXXXXXXXXXXXXXXXXX") @@ -344,7 +356,8 @@ class TestTaskExecutor(unittest.TestCase): ) te._get_connection = MagicMock(return_value=mock_connection) - te._get_action_handler = MagicMock(return_value=mock_action) + context = MagicMock() + te._get_action_handler_with_context = MagicMock(return_value=get_with_context_result(mock_action, context)) mock_action.run.return_value = dict(ansible_facts=dict()) res = te._execute() diff --git a/test/units/executor/test_task_queue_manager_callbacks.py b/test/units/executor/test_task_queue_manager_callbacks.py index b6b1159d..c63385dc 100644 --- a/test/units/executor/test_task_queue_manager_callbacks.py +++ b/test/units/executor/test_task_queue_manager_callbacks.py @@ -19,7 +19,7 @@ from __future__ import (absolute_import, division, print_function) from units.compat import unittest -from mock import MagicMock +from unittest.mock import MagicMock from ansible.executor.task_queue_manager import TaskQueueManager from ansible.playbook import Playbook diff --git a/test/units/executor/test_task_result.py b/test/units/executor/test_task_result.py index ee5c7198..8b79571f 100644 --- a/test/units/executor/test_task_result.py +++ b/test/units/executor/test_task_result.py @@ -20,7 +20,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type from units.compat import unittest -from mock import patch, MagicMock +from unittest.mock import patch, MagicMock from ansible.executor.task_result import TaskResult diff --git a/test/units/galaxy/test_api.py b/test/units/galaxy/test_api.py index 733f99b5..38011cc6 100644 --- a/test/units/galaxy/test_api.py +++ b/test/units/galaxy/test_api.py @@ -16,7 +16,7 @@ import tempfile import time from io import BytesIO, StringIO -from mock import MagicMock +from unittest.mock import MagicMock import ansible.constants as C from ansible import context @@ -75,6 +75,57 @@ def get_test_galaxy_api(url, version, token_ins=None, token_value=None, no_cache return api +def get_v3_collection_versions(namespace='namespace', name='collection'): + pagination_path = f"/api/galaxy/content/community/v3/plugin/{namespace}/content/community/collections/index/{namespace}/{name}/versions" + page_versions = (('1.0.0', '1.0.1',), ('1.0.2', '1.0.3',), ('1.0.4', '1.0.5'),) + responses = [ + {}, # TODO: initial response + ] + + first = f"{pagination_path}/?limit=100" + last = f"{pagination_path}/?limit=100&offset=200" + page_versions = [ + { + "versions": ('1.0.0', '1.0.1',), + "url": first, + }, + { + "versions": ('1.0.2', '1.0.3',), + "url": f"{pagination_path}/?limit=100&offset=100", + }, + { + "versions": ('1.0.4', '1.0.5'), + "url": last, + }, + ] + + previous = None + for page in range(0, len(page_versions)): + data = [] + + if page_versions[page]["url"] == last: + next_page = None + else: + next_page = page_versions[page + 1]["url"] + links = {"first": first, "last": last, "next": next_page, "previous": previous} + + for version in page_versions[page]["versions"]: + data.append( + { + "version": f"{version}", + "href": f"{pagination_path}/{version}/", + "created_at": "2022-05-13T15:55:58.913107Z", + "updated_at": "2022-05-13T15:55:58.913121Z", + "requires_ansible": ">=2.9.10" + } + ) + + responses.append({"meta": {"count": 6}, "links": links, "data": data}) + + previous = page_versions[page]["url"] + return responses + + def get_collection_versions(namespace='namespace', name='collection'): base_url = 'https://galaxy.server.com/api/v2/collections/{0}/{1}/'.format(namespace, name) versions_url = base_url + 'versions/' @@ -1149,6 +1200,35 @@ def test_cache_complete_pagination(cache_dir, monkeypatch): assert cached_versions == actual_versions +def test_cache_complete_pagination_v3(cache_dir, monkeypatch): + + responses = get_v3_collection_versions() + cache_file = os.path.join(cache_dir, 'api.json') + + api = get_test_galaxy_api('https://galaxy.server.com/api/', 'v3', no_cache=False) + + mock_open = MagicMock( + side_effect=[ + StringIO(to_text(json.dumps(r))) + for r in responses + ] + ) + monkeypatch.setattr(galaxy_api, 'open_url', mock_open) + + actual_versions = api.get_collection_versions('namespace', 'collection') + assert actual_versions == [u'1.0.0', u'1.0.1', u'1.0.2', u'1.0.3', u'1.0.4', u'1.0.5'] + + with open(cache_file) as fd: + final_cache = json.loads(fd.read()) + + cached_server = final_cache['galaxy.server.com:'] + cached_collection = cached_server['/api/v3/collections/namespace/collection/versions/'] + cached_versions = [r['version'] for r in cached_collection['results']] + + assert final_cache == api._cache + assert cached_versions == actual_versions + + def test_cache_flaky_pagination(cache_dir, monkeypatch): responses = get_collection_versions() diff --git a/test/units/galaxy/test_collection.py b/test/units/galaxy/test_collection.py index 53d042fe..e486daf6 100644 --- a/test/units/galaxy/test_collection.py +++ b/test/units/galaxy/test_collection.py @@ -15,7 +15,7 @@ import uuid from hashlib import sha256 from io import BytesIO -from mock import MagicMock, mock_open, patch +from unittest.mock import MagicMock, mock_open, patch import ansible.constants as C from ansible import context @@ -425,6 +425,23 @@ def test_missing_required_galaxy_key(galaxy_yml_dir): collection.concrete_artifact_manager._get_meta_from_src_dir(galaxy_yml_dir) +@pytest.mark.parametrize('galaxy_yml_dir', [b'namespace: test_namespace'], indirect=True) +def test_galaxy_yaml_no_mandatory_keys(galaxy_yml_dir): + expected = "The collection galaxy.yml at '%s/galaxy.yml' is missing the " \ + "following mandatory keys: authors, name, readme, version" % to_native(galaxy_yml_dir) + + with pytest.raises(ValueError, match=expected): + assert collection.concrete_artifact_manager._get_meta_from_src_dir(galaxy_yml_dir, require_build_metadata=False) == expected + + +@pytest.mark.parametrize('galaxy_yml_dir', [b'My life story is so very interesting'], indirect=True) +def test_galaxy_yaml_no_mandatory_keys_bad_yaml(galaxy_yml_dir): + expected = "The collection galaxy.yml at '%s/galaxy.yml' is incorrectly formatted." % to_native(galaxy_yml_dir) + + with pytest.raises(AnsibleError, match=expected): + collection.concrete_artifact_manager._get_meta_from_src_dir(galaxy_yml_dir) + + @pytest.mark.parametrize('galaxy_yml_dir', [b""" namespace: namespace name: collection diff --git a/test/units/galaxy/test_collection_install.py b/test/units/galaxy/test_collection_install.py index e34472f2..d83fe420 100644 --- a/test/units/galaxy/test_collection_install.py +++ b/test/units/galaxy/test_collection_install.py @@ -17,7 +17,7 @@ import tarfile import yaml from io import BytesIO, StringIO -from mock import MagicMock, patch +from unittest.mock import MagicMock, patch from unittest import mock import ansible.module_utils.six.moves.urllib.error as urllib_error @@ -181,13 +181,14 @@ def test_concrete_artifact_manager_scm_no_executable(monkeypatch): monkeypatch.setattr(collection.concrete_artifact_manager.subprocess, 'check_call', mock_subprocess_check_call) mock_mkdtemp = MagicMock(return_value='') monkeypatch.setattr(collection.concrete_artifact_manager, 'mkdtemp', mock_mkdtemp) + mock_get_bin_path = MagicMock(side_effect=[ValueError('Failed to find required executable')]) + monkeypatch.setattr(collection.concrete_artifact_manager, 'get_bin_path', mock_get_bin_path) error = re.escape( "Could not find git executable to extract the collection from the Git repository `https://github.com/org/repo`" ) - with mock.patch.dict(os.environ, {"PATH": ""}): - with pytest.raises(AnsibleError, match=error): - collection.concrete_artifact_manager._extract_collection_from_git(url, version, b'path') + with pytest.raises(AnsibleError, match=error): + collection.concrete_artifact_manager._extract_collection_from_git(url, version, b'path') @pytest.mark.parametrize( diff --git a/test/units/galaxy/test_token.py b/test/units/galaxy/test_token.py index 98dec5bf..24af3863 100644 --- a/test/units/galaxy/test_token.py +++ b/test/units/galaxy/test_token.py @@ -8,7 +8,7 @@ __metaclass__ = type import os import pytest -from mock import MagicMock +from unittest.mock import MagicMock import ansible.constants as C from ansible.cli.galaxy import GalaxyCLI, SERVER_DEF diff --git a/test/units/mock/path.py b/test/units/mock/path.py index dc51a143..c24ddf42 100644 --- a/test/units/mock/path.py +++ b/test/units/mock/path.py @@ -1,7 +1,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -from mock import MagicMock +from unittest.mock import MagicMock from ansible.utils.path import unfrackpath diff --git a/test/units/module_utils/basic/test_argument_spec.py b/test/units/module_utils/basic/test_argument_spec.py index 20bfb01e..211d65a2 100644 --- a/test/units/module_utils/basic/test_argument_spec.py +++ b/test/units/module_utils/basic/test_argument_spec.py @@ -12,7 +12,7 @@ import os import pytest -from mock import MagicMock +from units.compat.mock import MagicMock from ansible.module_utils import basic from ansible.module_utils.api import basic_auth_argument_spec, rate_limit_argument_spec, retry_argument_spec from ansible.module_utils.common import warnings diff --git a/test/units/module_utils/basic/test_filesystem.py b/test/units/module_utils/basic/test_filesystem.py index 92e2c46e..f09cecf4 100644 --- a/test/units/module_utils/basic/test_filesystem.py +++ b/test/units/module_utils/basic/test_filesystem.py @@ -9,7 +9,7 @@ __metaclass__ = type from units.mock.procenv import ModuleTestCase -from mock import patch, MagicMock +from units.compat.mock import patch, MagicMock from ansible.module_utils.six.moves import builtins realimport = builtins.__import__ diff --git a/test/units/module_utils/basic/test_get_module_path.py b/test/units/module_utils/basic/test_get_module_path.py index 2d0b8dd0..6ff4a3bc 100644 --- a/test/units/module_utils/basic/test_get_module_path.py +++ b/test/units/module_utils/basic/test_get_module_path.py @@ -9,7 +9,7 @@ __metaclass__ = type from units.mock.procenv import ModuleTestCase -from mock import patch +from units.compat.mock import patch from ansible.module_utils.six.moves import builtins realimport = builtins.__import__ diff --git a/test/units/module_utils/basic/test_imports.py b/test/units/module_utils/basic/test_imports.py index 79ab971f..d1a5f379 100644 --- a/test/units/module_utils/basic/test_imports.py +++ b/test/units/module_utils/basic/test_imports.py @@ -12,7 +12,7 @@ import sys from units.mock.procenv import ModuleTestCase from units.compat import unittest -from mock import patch +from units.compat.mock import patch from ansible.module_utils.six.moves import builtins realimport = builtins.__import__ diff --git a/test/units/module_utils/basic/test_platform_distribution.py b/test/units/module_utils/basic/test_platform_distribution.py index 6579bee9..3c1afb7d 100644 --- a/test/units/module_utils/basic/test_platform_distribution.py +++ b/test/units/module_utils/basic/test_platform_distribution.py @@ -9,7 +9,7 @@ __metaclass__ = type import pytest -from mock import patch +from units.compat.mock import patch from ansible.module_utils.six.moves import builtins diff --git a/test/units/module_utils/basic/test_selinux.py b/test/units/module_utils/basic/test_selinux.py index 600ff6b3..d8557685 100644 --- a/test/units/module_utils/basic/test_selinux.py +++ b/test/units/module_utils/basic/test_selinux.py @@ -11,7 +11,7 @@ import errno import json import pytest -from mock import mock_open, patch +from units.compat.mock import mock_open, patch from ansible.module_utils import basic from ansible.module_utils.common.text.converters import to_bytes diff --git a/test/units/module_utils/basic/test_set_cwd.py b/test/units/module_utils/basic/test_set_cwd.py index 77418601..159236b7 100644 --- a/test/units/module_utils/basic/test_set_cwd.py +++ b/test/units/module_utils/basic/test_set_cwd.py @@ -13,7 +13,7 @@ import tempfile import pytest -from mock import patch, MagicMock +from units.compat.mock import patch, MagicMock from ansible.module_utils._text import to_bytes from ansible.module_utils import basic diff --git a/test/units/module_utils/basic/test_tmpdir.py b/test/units/module_utils/basic/test_tmpdir.py index eec8f62c..818cb9b1 100644 --- a/test/units/module_utils/basic/test_tmpdir.py +++ b/test/units/module_utils/basic/test_tmpdir.py @@ -13,7 +13,7 @@ import tempfile import pytest -from mock import patch, MagicMock +from units.compat.mock import patch, MagicMock from ansible.module_utils._text import to_bytes from ansible.module_utils import basic diff --git a/test/units/module_utils/common/test_locale.py b/test/units/module_utils/common/test_locale.py index f8fea476..9d959860 100644 --- a/test/units/module_utils/common/test_locale.py +++ b/test/units/module_utils/common/test_locale.py @@ -5,7 +5,7 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type -from mock import MagicMock +from units.compat.mock import MagicMock from ansible.module_utils.common.locale import get_best_parsable_locale diff --git a/test/units/module_utils/common/test_sys_info.py b/test/units/module_utils/common/test_sys_info.py index 63101a81..18aafe53 100644 --- a/test/units/module_utils/common/test_sys_info.py +++ b/test/units/module_utils/common/test_sys_info.py @@ -9,7 +9,7 @@ __metaclass__ = type import pytest -from mock import patch +from units.compat.mock import patch from ansible.module_utils.six.moves import builtins diff --git a/test/units/module_utils/facts/base.py b/test/units/module_utils/facts/base.py index 23e620cb..33d3087b 100644 --- a/test/units/module_utils/facts/base.py +++ b/test/units/module_utils/facts/base.py @@ -20,7 +20,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type from units.compat import unittest -from mock import Mock, patch +from units.compat.mock import Mock, patch class BaseFactsTest(unittest.TestCase): diff --git a/test/units/module_utils/facts/hardware/test_linux.py b/test/units/module_utils/facts/hardware/test_linux.py index 1d584593..e3e07e78 100644 --- a/test/units/module_utils/facts/hardware/test_linux.py +++ b/test/units/module_utils/facts/hardware/test_linux.py @@ -19,7 +19,7 @@ __metaclass__ = type import os from units.compat import unittest -from mock import Mock, patch +from units.compat.mock import Mock, patch from ansible.module_utils.facts import timeout diff --git a/test/units/module_utils/facts/network/test_fc_wwn.py b/test/units/module_utils/facts/network/test_fc_wwn.py index 27d45234..32a3a43d 100644 --- a/test/units/module_utils/facts/network/test_fc_wwn.py +++ b/test/units/module_utils/facts/network/test_fc_wwn.py @@ -6,7 +6,7 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type from ansible.module_utils.facts.network import fc_wwn -from mock import Mock +from units.compat.mock import Mock # AIX lsdev diff --git a/test/units/module_utils/facts/network/test_generic_bsd.py b/test/units/module_utils/facts/network/test_generic_bsd.py index 79cc4815..afb698c5 100644 --- a/test/units/module_utils/facts/network/test_generic_bsd.py +++ b/test/units/module_utils/facts/network/test_generic_bsd.py @@ -18,7 +18,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -from mock import Mock +from units.compat.mock import Mock from units.compat import unittest from ansible.module_utils.facts.network import generic_bsd diff --git a/test/units/module_utils/facts/network/test_iscsi_get_initiator.py b/test/units/module_utils/facts/network/test_iscsi_get_initiator.py index 78e5c960..2048ba2a 100644 --- a/test/units/module_utils/facts/network/test_iscsi_get_initiator.py +++ b/test/units/module_utils/facts/network/test_iscsi_get_initiator.py @@ -6,7 +6,7 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type from ansible.module_utils.facts.network import iscsi -from mock import Mock +from units.compat.mock import Mock # AIX # lsattr -E -l iscsi0 diff --git a/test/units/module_utils/facts/other/test_facter.py b/test/units/module_utils/facts/other/test_facter.py index 517265d3..7466338e 100644 --- a/test/units/module_utils/facts/other/test_facter.py +++ b/test/units/module_utils/facts/other/test_facter.py @@ -19,7 +19,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -from mock import Mock, patch +from units.compat.mock import Mock, patch from .. base import BaseFactsTest diff --git a/test/units/module_utils/facts/other/test_ohai.py b/test/units/module_utils/facts/other/test_ohai.py index 38fb67f4..42a72d97 100644 --- a/test/units/module_utils/facts/other/test_ohai.py +++ b/test/units/module_utils/facts/other/test_ohai.py @@ -19,7 +19,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -from mock import Mock, patch +from units.compat.mock import Mock, patch from .. base import BaseFactsTest diff --git a/test/units/module_utils/facts/system/distribution/conftest.py b/test/units/module_utils/facts/system/distribution/conftest.py index 0282a7fc..d27b97f0 100644 --- a/test/units/module_utils/facts/system/distribution/conftest.py +++ b/test/units/module_utils/facts/system/distribution/conftest.py @@ -8,7 +8,7 @@ __metaclass__ = type import pytest -from mock import Mock +from units.compat.mock import Mock @pytest.fixture diff --git a/test/units/module_utils/facts/system/test_lsb.py b/test/units/module_utils/facts/system/test_lsb.py index 890bddb6..e2ed2ec0 100644 --- a/test/units/module_utils/facts/system/test_lsb.py +++ b/test/units/module_utils/facts/system/test_lsb.py @@ -19,7 +19,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -from mock import Mock, patch +from units.compat.mock import Mock, patch from .. base import BaseFactsTest diff --git a/test/units/module_utils/facts/test_ansible_collector.py b/test/units/module_utils/facts/test_ansible_collector.py index e1d60c3d..47d88df9 100644 --- a/test/units/module_utils/facts/test_ansible_collector.py +++ b/test/units/module_utils/facts/test_ansible_collector.py @@ -21,7 +21,7 @@ __metaclass__ = type # for testing from units.compat import unittest -from mock import Mock, patch +from units.compat.mock import Mock, patch from ansible.module_utils.facts import collector from ansible.module_utils.facts import ansible_collector diff --git a/test/units/module_utils/facts/test_collectors.py b/test/units/module_utils/facts/test_collectors.py index a6f12b56..c4806025 100644 --- a/test/units/module_utils/facts/test_collectors.py +++ b/test/units/module_utils/facts/test_collectors.py @@ -21,7 +21,7 @@ __metaclass__ = type import pytest -from mock import Mock, patch +from units.compat.mock import Mock, patch from . base import BaseFactsTest diff --git a/test/units/module_utils/facts/test_facts.py b/test/units/module_utils/facts/test_facts.py index a49616fc..c794f031 100644 --- a/test/units/module_utils/facts/test_facts.py +++ b/test/units/module_utils/facts/test_facts.py @@ -26,7 +26,7 @@ import pytest # for testing from units.compat import unittest -from mock import Mock, patch +from units.compat.mock import Mock, patch from ansible.module_utils import facts from ansible.module_utils.facts import hardware diff --git a/test/units/module_utils/facts/test_sysctl.py b/test/units/module_utils/facts/test_sysctl.py index 66336925..c369b610 100644 --- a/test/units/module_utils/facts/test_sysctl.py +++ b/test/units/module_utils/facts/test_sysctl.py @@ -26,7 +26,7 @@ import pytest # for testing from units.compat import unittest -from mock import patch, MagicMock, mock_open, Mock +from units.compat.mock import patch, MagicMock, mock_open, Mock from ansible.module_utils.facts.sysctl import get_sysctl diff --git a/test/units/module_utils/facts/test_utils.py b/test/units/module_utils/facts/test_utils.py index 70db0475..28cb5d31 100644 --- a/test/units/module_utils/facts/test_utils.py +++ b/test/units/module_utils/facts/test_utils.py @@ -18,7 +18,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type from units.compat import unittest -from mock import patch +from units.compat.mock import patch from ansible.module_utils.facts import utils diff --git a/test/units/module_utils/urls/test_Request.py b/test/units/module_utils/urls/test_Request.py index ebb6de56..648e46aa 100644 --- a/test/units/module_utils/urls/test_Request.py +++ b/test/units/module_utils/urls/test_Request.py @@ -13,7 +13,7 @@ from ansible.module_utils.urls import (Request, open_url, urllib_request, HAS_SS from ansible.module_utils.urls import SSLValidationHandler, HTTPSClientAuthHandler, RedirectHandlerFactory import pytest -from mock import call +from units.compat.mock import call if HAS_SSLCONTEXT: diff --git a/test/units/module_utils/urls/test_fetch_url.py b/test/units/module_utils/urls/test_fetch_url.py index 4869bb0f..56ff54a7 100644 --- a/test/units/module_utils/urls/test_fetch_url.py +++ b/test/units/module_utils/urls/test_fetch_url.py @@ -13,7 +13,7 @@ from ansible.module_utils.six.moves.http_client import HTTPMessage from ansible.module_utils.urls import fetch_url, urllib_error, ConnectionError, NoSSLError, httplib import pytest -from mock import MagicMock +from units.compat.mock import MagicMock class AnsibleModuleExit(Exception): diff --git a/test/units/modules/test_apt.py b/test/units/modules/test_apt.py index 78dbbade..20e056ff 100644 --- a/test/units/modules/test_apt.py +++ b/test/units/modules/test_apt.py @@ -4,8 +4,7 @@ __metaclass__ = type import collections import sys -import mock - +from units.compat.mock import Mock from units.compat import unittest try: @@ -41,14 +40,14 @@ class AptExpandPkgspecTestCase(unittest.TestCase): def test_pkgname_wildcard_version_wildcard(self): foo = ["apt*=1.0*"] - m_mock = mock.Mock() + m_mock = Mock() self.assertEqual( expand_pkgspec_from_fnmatches(m_mock, foo, self.fake_cache), ['apt', 'apt-utils']) def test_pkgname_expands(self): foo = ["apt*"] - m_mock = mock.Mock() + m_mock = Mock() self.assertEqual( expand_pkgspec_from_fnmatches(m_mock, foo, self.fake_cache), ["apt", "apt-utils"]) diff --git a/test/units/modules/test_apt_key.py b/test/units/modules/test_apt_key.py index 39339d76..37cd53b6 100644 --- a/test/units/modules/test_apt_key.py +++ b/test/units/modules/test_apt_key.py @@ -3,8 +3,7 @@ __metaclass__ = type import os -import mock - +from units.compat.mock import patch, Mock from units.compat import unittest from ansible.modules import apt_key @@ -16,11 +15,11 @@ def returnc(x): class AptKeyTestCase(unittest.TestCase): - @mock.patch.object(apt_key, 'apt_key_bin', '/usr/bin/apt-key') - @mock.patch.object(apt_key, 'lang_env', returnc) - @mock.patch.dict(os.environ, {'HTTP_PROXY': 'proxy.example.com'}) + @patch.object(apt_key, 'apt_key_bin', '/usr/bin/apt-key') + @patch.object(apt_key, 'lang_env', returnc) + @patch.dict(os.environ, {'HTTP_PROXY': 'proxy.example.com'}) def test_import_key_with_http_proxy(self): - m_mock = mock.Mock() + m_mock = Mock() m_mock.run_command.return_value = (0, '', '') apt_key.import_key( m_mock, keyring=None, keyserver='keyserver.example.com', diff --git a/test/units/modules/test_async_wrapper.py b/test/units/modules/test_async_wrapper.py index eacb9361..37b1fda3 100644 --- a/test/units/modules/test_async_wrapper.py +++ b/test/units/modules/test_async_wrapper.py @@ -11,7 +11,7 @@ import tempfile import pytest -from mock import patch, MagicMock +from units.compat.mock import patch, MagicMock from ansible.modules import async_wrapper from pprint import pprint diff --git a/test/units/modules/test_hostname.py b/test/units/modules/test_hostname.py index 804ecf74..9050fd04 100644 --- a/test/units/modules/test_hostname.py +++ b/test/units/modules/test_hostname.py @@ -5,7 +5,7 @@ import os import shutil import tempfile -from mock import patch, MagicMock, mock_open +from units.compat.mock import patch, MagicMock, mock_open from ansible.module_utils import basic from ansible.module_utils.common._utils import get_all_subclasses from ansible.modules import hostname diff --git a/test/units/modules/test_iptables.py b/test/units/modules/test_iptables.py index 5953334b..265e770a 100644 --- a/test/units/modules/test_iptables.py +++ b/test/units/modules/test_iptables.py @@ -1,7 +1,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type -from mock import patch +from units.compat.mock import patch from ansible.module_utils import basic from ansible.modules import iptables from units.modules.utils import AnsibleExitJson, AnsibleFailJson, ModuleTestCase, set_module_args diff --git a/test/units/modules/test_service_facts.py b/test/units/modules/test_service_facts.py index 3a180dc9..07f6827e 100644 --- a/test/units/modules/test_service_facts.py +++ b/test/units/modules/test_service_facts.py @@ -6,7 +6,7 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type from units.compat import unittest -from mock import patch +from units.compat.mock import patch from ansible.module_utils import basic from ansible.modules.service_facts import AIXScanService diff --git a/test/units/modules/utils.py b/test/units/modules/utils.py index 92f4ceab..6d169e36 100644 --- a/test/units/modules/utils.py +++ b/test/units/modules/utils.py @@ -4,7 +4,7 @@ __metaclass__ = type import json from units.compat import unittest -from mock import patch +from units.compat.mock import patch from ansible.module_utils import basic from ansible.module_utils._text import to_bytes diff --git a/test/units/parsing/test_dataloader.py b/test/units/parsing/test_dataloader.py index ed365b13..9ec49a8d 100644 --- a/test/units/parsing/test_dataloader.py +++ b/test/units/parsing/test_dataloader.py @@ -22,7 +22,7 @@ __metaclass__ = type import os from units.compat import unittest -from mock import patch, mock_open +from unittest.mock import patch, mock_open from ansible.errors import AnsibleParserError, yaml_strings, AnsibleFileNotFound from ansible.parsing.vault import AnsibleVaultError from ansible.module_utils._text import to_text diff --git a/test/units/parsing/vault/test_vault.py b/test/units/parsing/vault/test_vault.py index f92d451c..7afd3560 100644 --- a/test/units/parsing/vault/test_vault.py +++ b/test/units/parsing/vault/test_vault.py @@ -30,7 +30,7 @@ from binascii import hexlify import pytest from units.compat import unittest -from mock import patch, MagicMock +from unittest.mock import patch, MagicMock from ansible import errors from ansible.module_utils import six diff --git a/test/units/parsing/vault/test_vault_editor.py b/test/units/parsing/vault/test_vault_editor.py index 3f19b893..77509f08 100644 --- a/test/units/parsing/vault/test_vault_editor.py +++ b/test/units/parsing/vault/test_vault_editor.py @@ -27,7 +27,7 @@ from io import BytesIO, StringIO import pytest from units.compat import unittest -from mock import patch +from unittest.mock import patch from ansible import errors from ansible.parsing import vault diff --git a/test/units/playbook/role/test_include_role.py b/test/units/playbook/role/test_include_role.py index 79821b40..5e7625ba 100644 --- a/test/units/playbook/role/test_include_role.py +++ b/test/units/playbook/role/test_include_role.py @@ -20,7 +20,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type from units.compat import unittest -from mock import patch +from unittest.mock import patch from ansible.playbook import Play from ansible.playbook.role_include import IncludeRole diff --git a/test/units/playbook/role/test_role.py b/test/units/playbook/role/test_role.py index dacbc79c..5d47631f 100644 --- a/test/units/playbook/role/test_role.py +++ b/test/units/playbook/role/test_role.py @@ -22,7 +22,7 @@ __metaclass__ = type from collections.abc import Container from units.compat import unittest -from mock import patch, MagicMock +from unittest.mock import patch, MagicMock from ansible.errors import AnsibleError, AnsibleParserError from ansible.playbook.block import Block diff --git a/test/units/playbook/test_conditional.py b/test/units/playbook/test_conditional.py index 17284ca2..03ab3b7f 100644 --- a/test/units/playbook/test_conditional.py +++ b/test/units/playbook/test_conditional.py @@ -3,7 +3,7 @@ __metaclass__ = type from units.compat import unittest from units.mock.loader import DictDataLoader -from mock import MagicMock +from unittest.mock import MagicMock from ansible.template import Templar from ansible import errors diff --git a/test/units/playbook/test_helpers.py b/test/units/playbook/test_helpers.py index a921a727..d171bc38 100644 --- a/test/units/playbook/test_helpers.py +++ b/test/units/playbook/test_helpers.py @@ -22,7 +22,7 @@ __metaclass__ = type import os from units.compat import unittest -from mock import MagicMock +from unittest.mock import MagicMock from units.mock.loader import DictDataLoader from ansible import errors diff --git a/test/units/playbook/test_included_file.py b/test/units/playbook/test_included_file.py index bf79b927..7341dffa 100644 --- a/test/units/playbook/test_included_file.py +++ b/test/units/playbook/test_included_file.py @@ -23,7 +23,7 @@ import os import pytest -from mock import MagicMock +from unittest.mock import MagicMock from units.mock.loader import DictDataLoader from ansible.playbook.block import Block diff --git a/test/units/playbook/test_task.py b/test/units/playbook/test_task.py index 53a66705..070d7aa7 100644 --- a/test/units/playbook/test_task.py +++ b/test/units/playbook/test_task.py @@ -20,7 +20,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type from units.compat import unittest -from mock import patch +from unittest.mock import patch from ansible.playbook.task import Task from ansible.parsing.yaml import objects from ansible import errors diff --git a/test/units/plugins/action/test_action.py b/test/units/plugins/action/test_action.py index 70885181..78c6fe05 100644 --- a/test/units/plugins/action/test_action.py +++ b/test/units/plugins/action/test_action.py @@ -25,7 +25,7 @@ import re from ansible import constants as C from units.compat import unittest -from mock import patch, MagicMock, mock_open +from unittest.mock import patch, MagicMock, mock_open from ansible.errors import AnsibleError, AnsibleAuthenticationFailure from ansible.module_utils.six import text_type diff --git a/test/units/plugins/action/test_gather_facts.py b/test/units/plugins/action/test_gather_facts.py index e8a607b7..20225aa9 100644 --- a/test/units/plugins/action/test_gather_facts.py +++ b/test/units/plugins/action/test_gather_facts.py @@ -19,7 +19,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type from units.compat import unittest -from mock import MagicMock, patch +from unittest.mock import MagicMock, patch from ansible import constants as C from ansible.playbook.task import Task diff --git a/test/units/plugins/action/test_raw.py b/test/units/plugins/action/test_raw.py index da216385..33480516 100644 --- a/test/units/plugins/action/test_raw.py +++ b/test/units/plugins/action/test_raw.py @@ -22,7 +22,7 @@ import os from ansible.errors import AnsibleActionFail from units.compat import unittest -from mock import MagicMock, Mock +from unittest.mock import MagicMock, Mock from ansible.plugins.action.raw import ActionModule from ansible.playbook.task import Task from ansible.plugins.loader import connection_loader diff --git a/test/units/plugins/cache/test_cache.py b/test/units/plugins/cache/test_cache.py index d0a39f39..9fdff1f6 100644 --- a/test/units/plugins/cache/test_cache.py +++ b/test/units/plugins/cache/test_cache.py @@ -23,7 +23,7 @@ import os import shutil import tempfile -import mock +from unittest import mock from units.compat import unittest from ansible.errors import AnsibleError diff --git a/test/units/plugins/callback/test_callback.py b/test/units/plugins/callback/test_callback.py index 81ee3745..ccfa4658 100644 --- a/test/units/plugins/callback/test_callback.py +++ b/test/units/plugins/callback/test_callback.py @@ -25,7 +25,7 @@ import textwrap import types from units.compat import unittest -from mock import MagicMock +from unittest.mock import MagicMock from ansible.executor.task_result import TaskResult from ansible.inventory.host import Host diff --git a/test/units/plugins/connection/test_psrp.py b/test/units/plugins/connection/test_psrp.py index 73516cc6..38052e81 100644 --- a/test/units/plugins/connection/test_psrp.py +++ b/test/units/plugins/connection/test_psrp.py @@ -10,7 +10,7 @@ import pytest import sys from io import StringIO -from mock import MagicMock +from unittest.mock import MagicMock from ansible.playbook.play_context import PlayContext from ansible.plugins.loader import connection_loader diff --git a/test/units/plugins/connection/test_ssh.py b/test/units/plugins/connection/test_ssh.py index e7f4dd12..662dff91 100644 --- a/test/units/plugins/connection/test_ssh.py +++ b/test/units/plugins/connection/test_ssh.py @@ -27,7 +27,7 @@ import pytest from ansible import constants as C from ansible.errors import AnsibleAuthenticationFailure from units.compat import unittest -from mock import patch, MagicMock, PropertyMock +from unittest.mock import patch, MagicMock, PropertyMock from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound from ansible.module_utils.compat.selectors import SelectorKey, EVENT_READ from ansible.module_utils.six.moves import shlex_quote diff --git a/test/units/plugins/connection/test_winrm.py b/test/units/plugins/connection/test_winrm.py index c3245ccb..cb52814b 100644 --- a/test/units/plugins/connection/test_winrm.py +++ b/test/units/plugins/connection/test_winrm.py @@ -12,7 +12,7 @@ import pytest from io import StringIO -from mock import MagicMock +from unittest.mock import MagicMock from ansible.errors import AnsibleConnectionFailure from ansible.module_utils._text import to_bytes from ansible.playbook.play_context import PlayContext diff --git a/test/units/plugins/inventory/test_inventory.py b/test/units/plugins/inventory/test_inventory.py index 08148f8b..df246073 100644 --- a/test/units/plugins/inventory/test_inventory.py +++ b/test/units/plugins/inventory/test_inventory.py @@ -22,7 +22,7 @@ __metaclass__ = type import string import textwrap -import mock +from unittest import mock from ansible import constants as C from units.compat import unittest diff --git a/test/units/plugins/inventory/test_script.py b/test/units/plugins/inventory/test_script.py index 1a00946c..9f75199f 100644 --- a/test/units/plugins/inventory/test_script.py +++ b/test/units/plugins/inventory/test_script.py @@ -22,7 +22,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type import pytest -import mock +from unittest import mock from ansible import constants as C from ansible.errors import AnsibleError diff --git a/test/units/plugins/lookup/test_password.py b/test/units/plugins/lookup/test_password.py index c496ee6e..73b50418 100644 --- a/test/units/plugins/lookup/test_password.py +++ b/test/units/plugins/lookup/test_password.py @@ -32,7 +32,7 @@ import pytest from units.mock.loader import DictDataLoader from units.compat import unittest -from mock import mock_open, patch +from unittest.mock import mock_open, patch from ansible.errors import AnsibleError from ansible.module_utils.six import text_type from ansible.module_utils.six.moves import builtins diff --git a/test/units/plugins/strategy/test_linear.py b/test/units/plugins/strategy/test_linear.py index 3bce4856..2574e84f 100644 --- a/test/units/plugins/strategy/test_linear.py +++ b/test/units/plugins/strategy/test_linear.py @@ -7,7 +7,7 @@ __metaclass__ = type from units.compat import unittest -from mock import patch, MagicMock +from unittest.mock import patch, MagicMock from ansible.executor.play_iterator import PlayIterator from ansible.playbook import Playbook diff --git a/test/units/plugins/strategy/test_strategy.py b/test/units/plugins/strategy/test_strategy.py index 750e8069..bc4bb545 100644 --- a/test/units/plugins/strategy/test_strategy.py +++ b/test/units/plugins/strategy/test_strategy.py @@ -23,7 +23,7 @@ from units.mock.loader import DictDataLoader import uuid from units.compat import unittest -from mock import patch, MagicMock +from unittest.mock import patch, MagicMock from ansible.executor.process.worker import WorkerProcess from ansible.executor.task_queue_manager import TaskQueueManager from ansible.executor.task_result import TaskResult diff --git a/test/units/plugins/test_plugins.py b/test/units/plugins/test_plugins.py index 975fa420..46cd582d 100644 --- a/test/units/plugins/test_plugins.py +++ b/test/units/plugins/test_plugins.py @@ -23,7 +23,7 @@ __metaclass__ = type import os from units.compat import unittest -from mock import patch, MagicMock +from unittest.mock import patch, MagicMock from ansible.plugins.loader import PluginLoader, PluginPathContext diff --git a/test/units/template/test_templar.py b/test/units/template/test_templar.py index e922f95f..6747f768 100644 --- a/test/units/template/test_templar.py +++ b/test/units/template/test_templar.py @@ -22,7 +22,7 @@ __metaclass__ = type from jinja2.runtime import Context from units.compat import unittest -from mock import patch +from unittest.mock import patch from ansible import constants as C from ansible.errors import AnsibleError, AnsibleUndefinedVariable diff --git a/test/units/template/test_vars.py b/test/units/template/test_vars.py index 3e04ba2f..514104f2 100644 --- a/test/units/template/test_vars.py +++ b/test/units/template/test_vars.py @@ -20,7 +20,7 @@ from __future__ import (absolute_import, division, print_function) __metaclass__ = type from units.compat import unittest -from mock import MagicMock +from unittest.mock import MagicMock from ansible.template.vars import AnsibleJ2Vars diff --git a/test/units/test_no_tty.py b/test/units/test_no_tty.py new file mode 100644 index 00000000..290c0b92 --- /dev/null +++ b/test/units/test_no_tty.py @@ -0,0 +1,7 @@ +import sys + + +def test_no_tty(): + assert not sys.stdin.isatty() + assert not sys.stdout.isatty() + assert not sys.stderr.isatty() diff --git a/test/units/utils/collection_loader/test_collection_loader.py b/test/units/utils/collection_loader/test_collection_loader.py index 3ae04cbd..f7050dcd 100644 --- a/test/units/utils/collection_loader/test_collection_loader.py +++ b/test/units/utils/collection_loader/test_collection_loader.py @@ -17,7 +17,7 @@ from ansible.utils.collection_loader._collection_finder import ( _get_collection_name_from_path, _get_collection_role_path, _get_collection_metadata, _iter_modules_impl ) from ansible.utils.collection_loader._collection_config import _EventSource -from mock import MagicMock, NonCallableMagicMock, patch +from unittest.mock import MagicMock, NonCallableMagicMock, patch # fixture to ensure we always clean up the import stuff when we're done diff --git a/test/units/utils/display/test_broken_cowsay.py b/test/units/utils/display/test_broken_cowsay.py index e93065d8..d888010a 100644 --- a/test/units/utils/display/test_broken_cowsay.py +++ b/test/units/utils/display/test_broken_cowsay.py @@ -8,7 +8,7 @@ __metaclass__ = type from ansible.utils.display import Display -from mock import MagicMock +from unittest.mock import MagicMock def test_display_with_fake_cowsay_binary(capsys, mocker): diff --git a/test/units/utils/test_display.py b/test/units/utils/test_display.py index 8807b816..4883a5be 100644 --- a/test/units/utils/test_display.py +++ b/test/units/utils/test_display.py @@ -5,7 +5,7 @@ from __future__ import absolute_import, division, print_function __metaclass__ = type -from mock import MagicMock +from unittest.mock import MagicMock import pytest diff --git a/test/units/utils/test_vars.py b/test/units/utils/test_vars.py index 1df0eab3..9be33de4 100644 --- a/test/units/utils/test_vars.py +++ b/test/units/utils/test_vars.py @@ -22,7 +22,7 @@ __metaclass__ = type from collections import defaultdict -import mock +from unittest import mock from units.compat import unittest from ansible.errors import AnsibleError diff --git a/test/units/vars/test_variable_manager.py b/test/units/vars/test_variable_manager.py index fa68fd3b..67ec120b 100644 --- a/test/units/vars/test_variable_manager.py +++ b/test/units/vars/test_variable_manager.py @@ -22,7 +22,7 @@ __metaclass__ = type import os from units.compat import unittest -from mock import MagicMock, patch +from unittest.mock import MagicMock, patch from ansible.inventory.manager import InventoryManager from ansible.module_utils.six import iteritems from ansible.playbook.play import Play |