ansible-test - fix coverage for test modules (#84366)HEADdevel

Fixes the coverage path translation for modules located in integration
test paths. Instead of trying to match by the unique temporary path name
that the module is executed as, the reporting tool will translate it to
the static path that the module is actually located under.

9 files changed, 83 insertions, 5 deletions

diff --git a/changelogs/fragments/ansible-test-coverage-test-files.yml b/changelogs/fragments/ansible-test-coverage-test-files.yml
new file mode 100644
index 0000000000..28b35e6cc3
--- /dev/null
+++ b/changelogs/fragments/ansible-test-coverage-test-files.yml
@@ -0,0 +1,4 @@
+bugfixes:
+  - >-
+    ansible-test - Fix up coverage reporting to properly translate the temporary path of integration test modules to
+    the expected static test module path.
diff --git a/test/integration/targets/ansible-test-coverage-windows/aliases b/test/integration/targets/ansible-test-coverage-windows/aliases
new file mode 100644
index 0000000000..b7a6b16541
--- /dev/null
+++ b/test/integration/targets/ansible-test-coverage-windows/aliases
@@ -0,0 +1,3 @@
+shippable/windows/group1
+windows
+needs/target/collection
diff --git a/test/integration/targets/ansible-test-coverage-windows/ansible_collections/ns/col/plugins/modules/win_collection.ps1 b/test/integration/targets/ansible-test-coverage-windows/ansible_collections/ns/col/plugins/modules/win_collection.ps1
new file mode 100644
index 0000000000..53b2f2da3b
--- /dev/null
+++ b/test/integration/targets/ansible-test-coverage-windows/ansible_collections/ns/col/plugins/modules/win_collection.ps1
@@ -0,0 +1,6 @@
+#!powershell
+
+#AnsibleRequires -CSharpUtil Ansible.Basic
+
+$module = [Ansible.Basic.AnsibleModule]::Create($args, @{})
+$module.ExitJson()
diff --git a/test/integration/targets/ansible-test-coverage-windows/ansible_collections/ns/col/tests/integration/targets/win_collection/library/test_win_collection.ps1 b/test/integration/targets/ansible-test-coverage-windows/ansible_collections/ns/col/tests/integration/targets/win_collection/library/test_win_collection.ps1
new file mode 100644
index 0000000000..53b2f2da3b
--- /dev/null
+++ b/test/integration/targets/ansible-test-coverage-windows/ansible_collections/ns/col/tests/integration/targets/win_collection/library/test_win_collection.ps1
@@ -0,0 +1,6 @@
+#!powershell
+
+#AnsibleRequires -CSharpUtil Ansible.Basic
+
+$module = [Ansible.Basic.AnsibleModule]::Create($args, @{})
+$module.ExitJson()
diff --git a/test/integration/targets/ansible-test-coverage-windows/ansible_collections/ns/col/tests/integration/targets/win_collection/tasks/main.yml b/test/integration/targets/ansible-test-coverage-windows/ansible_collections/ns/col/tests/integration/targets/win_collection/tasks/main.yml
new file mode 100644
index 0000000000..6196b768c6
--- /dev/null
+++ b/test/integration/targets/ansible-test-coverage-windows/ansible_collections/ns/col/tests/integration/targets/win_collection/tasks/main.yml
@@ -0,0 +1,5 @@
+- name: run module in collection to test coverage for collection plugins
+  win_collection:
+
+- name: run module in library adjacent to test coverage for test plugins
+  test_win_collection:
diff --git a/test/integration/targets/ansible-test-coverage-windows/runme.sh b/test/integration/targets/ansible-test-coverage-windows/runme.sh
new file mode 100755
index 0000000000..70593e017a
--- /dev/null
+++ b/test/integration/targets/ansible-test-coverage-windows/runme.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+TEST_PATH="${PWD}/test-coverage.py"
+
+source ../collection/setup.sh
+cp "${INVENTORY_PATH}" tests/integration/inventory.winrm
+
+set -x
+
+# common args for all tests
+common=(--venv --color --truncate 0 "${@}")
+
+# run command that generates coverage data for Windows
+ansible-test windows-integration win_collection "${common[@]}" --coverage
+
+# report on code coverage in all supported formats
+ansible-test coverage report "${common[@]}"
+
+# test we covered the 2 files we expect to have been covered and their lines
+python "${TEST_PATH}"
diff --git a/test/integration/targets/ansible-test-coverage-windows/test-coverage.py b/test/integration/targets/ansible-test-coverage-windows/test-coverage.py
new file mode 100644
index 0000000000..98dbca7437
--- /dev/null
+++ b/test/integration/targets/ansible-test-coverage-windows/test-coverage.py
@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+import json
+import os
+import os.path
+
+
+def main() -> None:
+    collection_root = os.getcwd()
+    print(f"Running windows-integration coverage test in '{collection_root}'")
+
+    result_path = os.path.join(collection_root, "tests", "output", "coverage", "coverage-powershell")
+    module_path = os.path.join(collection_root, "plugins", "modules", "win_collection.ps1")
+    test_path = os.path.join(collection_root, "tests", "integration", "targets", "win_collection", "library", "test_win_collection.ps1")
+    with open(result_path, mode="rb") as fd:
+        data = json.load(fd)
+
+    for path, result in data.items():
+        print(f"Testing result for path '{path}' -> {result!r}")
+        assert path in [module_path, test_path], f"Found unexpected coverage result path '{path}'"
+        assert result == {'5': 1, '6': 1}, "Coverage result did not pick up a hit on lines 5 and 6"
+
+    assert len(data) == 2, f"Expected coverage results for 2 files but got {len(data)}"
+
+
+if __name__ == '__main__':
+    main()
diff --git a/test/integration/targets/collection/setup.sh b/test/integration/targets/collection/setup.sh
index 74466555e1..346a21ca9a 100755
--- a/test/integration/targets/collection/setup.sh
+++ b/test/integration/targets/collection/setup.sh
@@ -12,6 +12,9 @@
 #
 # 3) Sanity tests which are multi-version require an ignore entry per Python version.
 #    This script replicates these ignore entries for each supported Python version based on the ignored path.
+#
+# 4) Windows tests need access to the ansible.windows vendored collection.
+#    This script copies any of the existing collections in ANSIBLE_COLLECTIONS_PATH to the temporary directory.
 
 set -eu -o pipefail
 
@@ -26,4 +29,8 @@ trap 'rm -rf "${WORK_DIR}"' EXIT
 cp -a "${TEST_DIR}/ansible_collections" "${WORK_DIR}"
 cd "${WORK_DIR}/ansible_collections/ns/${COLLECTION_NAME:-col}"
 
+if [ "${ANSIBLE_COLLECTIONS_PATH:+set}" = "set" ]; then
+    cp -aL "${ANSIBLE_COLLECTIONS_PATH}"/ansible_collections/* "${WORK_DIR}/ansible_collections"
+fi
+
 "${TEST_DIR}/../collection/update-ignore.py"
diff --git a/test/lib/ansible_test/_internal/commands/coverage/__init__.py b/test/lib/ansible_test/_internal/commands/coverage/__init__.py
index c4c5f09e82..6c6e8cdaba 100644
--- a/test/lib/ansible_test/_internal/commands/coverage/__init__.py
+++ b/test/lib/ansible_test/_internal/commands/coverage/__init__.py
@@ -293,6 +293,11 @@ def sanitize_filename(
         new_name = re.sub('^.*/ansible_modlib.zip/ansible/', ansible_path, filename)
         display.info('%s -> %s' % (filename, new_name), verbosity=3)
         filename = new_name
+    elif integration_temp_path in filename:
+        # Rewrite the path of code running from an integration test temporary directory.
+        new_name = re.sub(r'^.*' + re.escape(integration_temp_path) + '[^/]+/', root_path, filename)
+        display.info('%s -> %s' % (filename, new_name), verbosity=3)
+        filename = new_name
     elif collection_search_re and collection_search_re.search(filename):
         new_name = os.path.abspath(collection_sub_re.sub('', filename))
         display.info('%s -> %s' % (filename, new_name), verbosity=3)
@@ -328,11 +333,6 @@ def sanitize_filename(
         new_name = re.sub('^(/.*?)?/root/ansible/', root_path, filename)
         display.info('%s -> %s' % (filename, new_name), verbosity=3)
         filename = new_name
-    elif integration_temp_path in filename:
-        # Rewrite the path of code running from an integration test temporary directory.
-        new_name = re.sub(r'^.*' + re.escape(integration_temp_path) + '[^/]+/', root_path, filename)
-        display.info('%s -> %s' % (filename, new_name), verbosity=3)
-        filename = new_name
 
     filename = os.path.abspath(filename)  # make sure path is absolute (will be relative if previously exported)