Add code coverage target analysis to ansible-test. (#67141)

* Refactor coverage file enumeration.
* Relocate sanitize_filename function.
* Support sets when writing JSON files.
* Generalize setting of info_stderr mode.
* Split out coverage path checking.
* Split out collection regex logic.
* Improve sanitize_filename type hints and docs.
* Clean up coverage erase command.
* Fix docs and type hints for initialize_coverage.
* Update type hints on CoverageConfig.
* Split out logic for finding modules.
* Split out arc enumeration.
* Split out powershell coverage enumeration.
* Raise verbosity level of empty coverage warnings.
* Add code coverage target analysis to ansible-test.

15 files changed, 944 insertions, 187 deletions

diff --git a/changelogs/fragments/ansible-test-coverage-analyze-targets.yml b/changelogs/fragments/ansible-test-coverage-analyze-targets.yml
new file mode 100644
index 0000000000..72340f0b7f
--- /dev/null
+++ b/changelogs/fragments/ansible-test-coverage-analyze-targets.yml
@@ -0,0 +1,2 @@
+minor_changes:
+    - "ansible-test - Added a ``ansible-test coverage analyze targets`` command to analyze integration test code coverage by test target."
diff --git a/test/lib/ansible_test/_internal/cli.py b/test/lib/ansible_test/_internal/cli.py
index acf8d72272..c2c2f9d5f6 100644
--- a/test/lib/ansible_test/_internal/cli.py
+++ b/test/lib/ansible_test/_internal/cli.py
@@ -12,6 +12,8 @@ from .init import (
     CURRENT_RLIMIT_NOFILE,
 )
 
+from . import types as t
+
 from .util import (
     ApplicationError,
     display,
@@ -42,7 +44,6 @@ from .executor import (
 )
 
 from .config import (
-    IntegrationConfig,
     PosixIntegrationConfig,
     WindowsIntegrationConfig,
     NetworkIntegrationConfig,
@@ -113,11 +114,34 @@ from .coverage.xml import (
     command_coverage_xml,
 )
 
+from .coverage.analyze.targets.generate import (
+    command_coverage_analyze_targets_generate,
+    CoverageAnalyzeTargetsGenerateConfig,
+)
+
+from .coverage.analyze.targets.expand import (
+    command_coverage_analyze_targets_expand,
+    CoverageAnalyzeTargetsExpandConfig,
+)
+
+from .coverage.analyze.targets.combine import (
+    command_coverage_analyze_targets_combine,
+    CoverageAnalyzeTargetsCombineConfig,
+)
+
+from .coverage.analyze.targets.missing import (
+    command_coverage_analyze_targets_missing,
+    CoverageAnalyzeTargetsMissingConfig,
+)
+
 from .coverage import (
     COVERAGE_GROUPS,
     CoverageConfig,
 )
 
+if t.TYPE_CHECKING:
+    import argparse as argparse_module
+
 
 def main():
     """Main program function."""
@@ -131,7 +155,7 @@ def main():
         display.truncate = config.truncate
         display.redact = config.redact
         display.color = config.color
-        display.info_stderr = (isinstance(config, SanityConfig) and config.lint) or (isinstance(config, IntegrationConfig) and config.list_targets)
+        display.info_stderr = config.info_stderr
         check_startup()
         check_delegation_args(config)
         configure_timeout(config)
@@ -147,6 +171,7 @@ def main():
             delegate_args = (ex.exclude, ex.require, ex.integration_targets)
 
         if delegate_args:
+            # noinspection PyTypeChecker
             delegate(config, *delegate_args)
 
         display.review_warnings()
@@ -513,6 +538,8 @@ def parse_args():
     coverage_subparsers = coverage.add_subparsers(metavar='COMMAND')
     coverage_subparsers.required = True  # work-around for python 3 bug which makes subparsers optional
 
+    add_coverage_analyze(coverage_subparsers, coverage_common)
+
     coverage_combine = coverage_subparsers.add_parser('combine',
                                                       parents=[coverage_common],
                                                       help='combine coverage data and rewrite remote paths')
@@ -608,6 +635,129 @@ def parse_args():
     return args
 
 
+# noinspection PyProtectedMember
+def add_coverage_analyze(coverage_subparsers, coverage_common):  # type: (argparse_module._SubParsersAction, argparse_module.ArgumentParser) -> None
+    """Add the `coverage analyze` subcommand."""
+    analyze = coverage_subparsers.add_parser(
+        'analyze',
+        help='analyze collected coverage data',
+    )
+
+    analyze_subparsers = analyze.add_subparsers(metavar='COMMAND')
+    analyze_subparsers.required = True  # work-around for python 3 bug which makes subparsers optional
+
+    targets = analyze_subparsers.add_parser(
+        'targets',
+        help='analyze integration test target coverage',
+    )
+
+    targets_subparsers = targets.add_subparsers(metavar='COMMAND')
+    targets_subparsers.required = True  # work-around for python 3 bug which makes subparsers optional
+
+    targets_generate = targets_subparsers.add_parser(
+        'generate',
+        parents=[coverage_common],
+        help='aggregate coverage by integration test target',
+    )
+
+    targets_generate.set_defaults(
+        func=command_coverage_analyze_targets_generate,
+        config=CoverageAnalyzeTargetsGenerateConfig,
+    )
+
+    targets_generate.add_argument(
+        'input_dir',
+        nargs='?',
+        help='directory to read coverage from',
+    )
+
+    targets_generate.add_argument(
+        'output_file',
+        help='output file for aggregated coverage',
+    )
+
+    targets_expand = targets_subparsers.add_parser(
+        'expand',
+        parents=[coverage_common],
+        help='expand target names from integers in aggregated coverage',
+    )
+
+    targets_expand.set_defaults(
+        func=command_coverage_analyze_targets_expand,
+        config=CoverageAnalyzeTargetsExpandConfig,
+    )
+
+    targets_expand.add_argument(
+        'input_file',
+        help='input file to read aggregated coverage from',
+    )
+
+    targets_expand.add_argument(
+        'output_file',
+        help='output file to write expanded coverage to',
+    )
+
+    targets_combine = targets_subparsers.add_parser(
+        'combine',
+        parents=[coverage_common],
+        help='combine multiple aggregated coverage files',
+    )
+
+    targets_combine.set_defaults(
+        func=command_coverage_analyze_targets_combine,
+        config=CoverageAnalyzeTargetsCombineConfig,
+    )
+
+    targets_combine.add_argument(
+        'input_file',
+        nargs='+',
+        help='input file to read aggregated coverage from',
+    )
+
+    targets_combine.add_argument(
+        'output_file',
+        help='output file to write aggregated coverage to',
+    )
+
+    targets_missing = targets_subparsers.add_parser(
+        'missing',
+        parents=[coverage_common],
+        help='identify coverage in one file missing in another',
+    )
+
+    targets_missing.set_defaults(
+        func=command_coverage_analyze_targets_missing,
+        config=CoverageAnalyzeTargetsMissingConfig,
+    )
+
+    targets_missing.add_argument(
+        'from_file',
+        help='input file containing aggregated coverage',
+    )
+
+    targets_missing.add_argument(
+        'to_file',
+        help='input file containing aggregated coverage',
+    )
+
+    targets_missing.add_argument(
+        'output_file',
+        help='output file to write aggregated coverage to',
+    )
+
+    targets_missing.add_argument(
+        '--only-gaps',
+        action='store_true',
+        help='report only arcs/lines not hit by any target',
+    )
+
+    targets_missing.add_argument(
+        '--only-exists',
+        action='store_true',
+        help='limit results to files that exist',
+    )
+
+
 def add_lint(parser):
     """
     :type parser: argparse.ArgumentParser
@@ -923,6 +1073,6 @@ def complete_sanity_test(prefix, parsed_args, **_):
     """
     del parsed_args
 
-    tests = sorted(t.name for t in sanity_get_tests())
+    tests = sorted(test.name for test in sanity_get_tests())
 
     return [i for i in tests if i.startswith(prefix)]
diff --git a/test/lib/ansible_test/_internal/config.py b/test/lib/ansible_test/_internal/config.py
index 4a7e63606a..1979ef735b 100644
--- a/test/lib/ansible_test/_internal/config.py
+++ b/test/lib/ansible_test/_internal/config.py
@@ -225,6 +225,8 @@ class SanityConfig(TestConfig):
         else:
             self.base_branch = ''
 
+        self.info_stderr = self.lint
+
 
 class IntegrationConfig(TestConfig):
     """Configuration for the integration command."""
@@ -260,6 +262,7 @@ class IntegrationConfig(TestConfig):
 
         if self.list_targets:
             self.explain = True
+            self.info_stderr = True
 
     def get_ansible_config(self):  # type: () -> str
         """Return the path to the Ansible config for the given config."""
diff --git a/test/lib/ansible_test/_internal/coverage/__init__.py b/test/lib/ansible_test/_internal/coverage/__init__.py
index de94f36849..4da90551d1 100644
--- a/test/lib/ansible_test/_internal/coverage/__init__.py
+++ b/test/lib/ansible_test/_internal/coverage/__init__.py
@@ -3,17 +3,28 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import os
+import re
 
 from .. import types as t
 
+from ..encoding import (
+    to_bytes,
+)
+
+from ..io import (
+    read_json_file,
+)
+
 from ..util import (
     ApplicationError,
     common_environment,
+    display,
     ANSIBLE_TEST_DATA_ROOT,
 )
 
 from ..util_common import (
     intercept_command,
+    ResultType,
 )
 
 from ..config import (
@@ -25,16 +36,24 @@ from ..executor import (
     install_command_requirements,
 )
 
+from .. target import (
+    walk_module_targets,
+)
+
+from ..data import (
+    data_context,
+)
+
+if t.TYPE_CHECKING:
+    import coverage as coverage_module
+
 COVERAGE_GROUPS = ('command', 'target', 'environment', 'version')
 COVERAGE_CONFIG_PATH = os.path.join(ANSIBLE_TEST_DATA_ROOT, 'coveragerc')
 COVERAGE_OUTPUT_FILE_NAME = 'coverage'
 
 
-def initialize_coverage(args):
-    """
-    :type args: CoverageConfig
-    :rtype: coverage
-    """
+def initialize_coverage(args):  # type: (CoverageConfig) -> coverage_module
+    """Delegate execution if requested, install requirements, then import and return the coverage module. Raises an exception if coverage is not available."""
     if args.delegate:
         raise Delegate()
 
@@ -62,15 +81,206 @@ def run_coverage(args, output_file, command, cmd):  # type: (CoverageConfig, str
     intercept_command(args, target_name='coverage', env=env, cmd=cmd, disable_coverage=True)
 
 
+def get_python_coverage_files():  # type: () -> t.List[str]
+    """Return the list of Python coverage file paths."""
+    return get_coverage_files('python')
+
+
+def get_powershell_coverage_files():  # type: () -> t.List[str]
+    """Return the list of PowerShell coverage file paths."""
+    return get_coverage_files('powershell')
+
+
+def get_coverage_files(language):  # type: (str) -> t.List[str]
+    """Return the list of coverage file paths for the given language."""
+    coverage_dir = ResultType.COVERAGE.path
+    coverage_files = [os.path.join(coverage_dir, f) for f in os.listdir(coverage_dir)
+                      if '=coverage.' in f and '=%s' % language in f]
+
+    return coverage_files
+
+
+def get_collection_path_regexes():  # type: () -> t.Tuple[t.Optional[t.Pattern], t.Optional[t.Pattern]]
+    """Return a pair of regexes used for identifying and manipulating collection paths."""
+    if data_context().content.collection:
+        collection_search_re = re.compile(r'/%s/' % data_context().content.collection.directory)
+        collection_sub_re = re.compile(r'^.*?/%s/' % data_context().content.collection.directory)
+    else:
+        collection_search_re = None
+        collection_sub_re = None
+
+    return collection_search_re, collection_sub_re
+
+
+def get_python_modules():  # type: () -> t.Dict[str, str]
+    """Return a dictionary of Ansible module names and their paths."""
+    return dict((target.module, target.path) for target in list(walk_module_targets()) if target.path.endswith('.py'))
+
+
+def enumerate_python_arcs(
+        path,  # type: str
+        coverage,  # type: coverage_module
+        modules,  # type: t.Dict[str, str]
+        collection_search_re,  # type: t.Optional[t.Pattern]
+        collection_sub_re,  # type: t.Optional[t.Pattern]
+):  # type: (...) -> t.Generator[t.Tuple[str, t.Set[t.Tuple[int, int]]]]
+    """Enumerate Python code coverage arcs in the given file."""
+    if os.path.getsize(path) == 0:
+        display.warning('Empty coverage file: %s' % path, verbosity=2)
+        return
+
+    original = coverage.CoverageData()
+
+    try:
+        original.read_file(path)
+    except Exception as ex:  # pylint: disable=locally-disabled, broad-except
+        display.error(u'%s' % ex)
+        return
+
+    for filename in original.measured_files():
+        arcs = original.arcs(filename)
+
+        if not arcs:
+            # This is most likely due to using an unsupported version of coverage.
+            display.warning('No arcs found for "%s" in coverage file: %s' % (filename, path))
+            continue
+
+        filename = sanitize_filename(filename, modules=modules, collection_search_re=collection_search_re, collection_sub_re=collection_sub_re)
+
+        if not filename:
+            continue
+
+        yield filename, set(arcs)
+
+
+def enumerate_powershell_lines(path):  # type: (str) -> t.Generator[t.Tuple[str, t.Dict[int, int]]]
+    """Enumerate PowerShell code coverage lines in the given file."""
+    if os.path.getsize(path) == 0:
+        display.warning('Empty coverage file: %s' % path, verbosity=2)
+        return
+
+    try:
+        coverage_run = read_json_file(path)
+    except Exception as ex:  # pylint: disable=locally-disabled, broad-except
+        display.error(u'%s' % ex)
+        return
+
+    for filename, hits in coverage_run.items():
+        filename = sanitize_filename(filename)
+
+        if not filename:
+            continue
+
+        # PowerShell unpacks arrays if there's only a single entry so this is a defensive check on that
+        if not isinstance(hits, list):
+            hits = [hits]
+
+        hits = dict((hit['Line'], hit['HitCount']) for hit in hits if hit)
+
+        yield filename, hits
+
+
+def sanitize_filename(
+        filename,  # type: str
+        modules=None,  # type: t.Optional[t.Dict[str, str]]
+        collection_search_re=None,  # type: t.Optional[t.Pattern]
+        collection_sub_re=None,  # type: t.Optional[t.Pattern]
+):  # type: (...) -> t.Optional[str]
+    """Convert the given code coverage path to a local absolute path and return its, or None if the path is not valid."""
+    ansible_path = os.path.abspath('lib/ansible/') + '/'
+    root_path = data_context().content.root + '/'
+    integration_temp_path = os.path.sep + os.path.join(ResultType.TMP.relative_path, 'integration') + os.path.sep
+
+    if modules is None:
+        modules = {}
+
+    if '/ansible_modlib.zip/ansible/' in filename:
+        # Rewrite the module_utils path from the remote host to match the controller. Ansible 2.6 and earlier.
+        new_name = re.sub('^.*/ansible_modlib.zip/ansible/', ansible_path, filename)
+        display.info('%s -> %s' % (filename, new_name), verbosity=3)
+        filename = new_name
+    elif collection_search_re and collection_search_re.search(filename):
+        new_name = os.path.abspath(collection_sub_re.sub('', filename))
+        display.info('%s -> %s' % (filename, new_name), verbosity=3)
+        filename = new_name
+    elif re.search(r'/ansible_[^/]+_payload\.zip/ansible/', filename):
+        # Rewrite the module_utils path from the remote host to match the controller. Ansible 2.7 and later.
+        new_name = re.sub(r'^.*/ansible_[^/]+_payload\.zip/ansible/', ansible_path, filename)
+        display.info('%s -> %s' % (filename, new_name), verbosity=3)
+        filename = new_name
+    elif '/ansible_module_' in filename:
+        # Rewrite the module path from the remote host to match the controller. Ansible 2.6 and earlier.
+        module_name = re.sub('^.*/ansible_module_(?P<module>.*).py$', '\\g<module>', filename)
+        if module_name not in modules:
+            display.warning('Skipping coverage of unknown module: %s' % module_name)
+            return None
+        new_name = os.path.abspath(modules[module_name])
+        display.info('%s -> %s' % (filename, new_name), verbosity=3)
+        filename = new_name
+    elif re.search(r'/ansible_[^/]+_payload(_[^/]+|\.zip)/__main__\.py$', filename):
+        # Rewrite the module path from the remote host to match the controller. Ansible 2.7 and later.
+        # AnsiballZ versions using zipimporter will match the `.zip` portion of the regex.
+        # AnsiballZ versions not using zipimporter will match the `_[^/]+` portion of the regex.
+        module_name = re.sub(r'^.*/ansible_(?P<module>[^/]+)_payload(_[^/]+|\.zip)/__main__\.py$',
+                             '\\g<module>', filename).rstrip('_')
+        if module_name not in modules:
+            display.warning('Skipping coverage of unknown module: %s' % module_name)
+            return None
+        new_name = os.path.abspath(modules[module_name])
+        display.info('%s -> %s' % (filename, new_name), verbosity=3)
+        filename = new_name
+    elif re.search('^(/.*?)?/root/ansible/', filename):
+        # Rewrite the path of code running on a remote host or in a docker container as root.
+        new_name = re.sub('^(/.*?)?/root/ansible/', root_path, filename)
+        display.info('%s -> %s' % (filename, new_name), verbosity=3)
+        filename = new_name
+    elif integration_temp_path in filename:
+        # Rewrite the path of code running from an integration test temporary directory.
+        new_name = re.sub(r'^.*' + re.escape(integration_temp_path) + '[^/]+/', root_path, filename)
+        display.info('%s -> %s' % (filename, new_name), verbosity=3)
+        filename = new_name
+
+    return filename
+
+
 class CoverageConfig(EnvironmentConfig):
     """Configuration for the coverage command."""
-    def __init__(self, args):
-        """
-        :type args: any
-        """
+    def __init__(self, args):  # type: (t.Any) -> None
         super(CoverageConfig, self).__init__(args, 'coverage')
 
         self.group_by = frozenset(args.group_by) if 'group_by' in args and args.group_by else set()  # type: t.FrozenSet[str]
         self.all = args.all if 'all' in args else False  # type: bool
         self.stub = args.stub if 'stub' in args else False  # type: bool
         self.coverage = False  # temporary work-around to support intercept_command in cover.py
+
+
+class PathChecker:
+    """Checks code coverage paths to verify they are valid and reports on the findings."""
+    def __init__(self, args, collection_search_re=None):  # type: (CoverageConfig, t.Optional[t.Pattern]) -> None
+        self.args = args
+        self.collection_search_re = collection_search_re
+        self.invalid_paths = []
+        self.invalid_path_chars = 0
+
+    def check_path(self, path):  # type: (str) -> bool
+        """Return True if the given coverage path is valid, otherwise display a warning and return False."""
+        if os.path.isfile(to_bytes(path)):
+            return True
+
+        if self.collection_search_re and self.collection_search_re.search(path) and os.path.basename(path) == '__init__.py':
+            # the collection loader uses implicit namespace packages, so __init__.py does not need to exist on disk
+            # coverage is still reported for these non-existent files, but warnings are not needed
+            return False
+
+        self.invalid_paths.append(path)
+        self.invalid_path_chars += len(path)
+
+        if self.args.verbosity > 1:
+            display.warning('Invalid coverage path: %s' % path)
+
+        return False
+
+    def report(self):  # type: () -> None
+        """Display a warning regarding invalid paths if any were found."""
+        if self.invalid_paths:
+            display.warning('Ignored %d characters from %d invalid coverage path(s).' % (self.invalid_path_chars, len(self.invalid_paths)))
diff --git a/test/lib/ansible_test/_internal/coverage/analyze/__init__.py b/test/lib/ansible_test/_internal/coverage/analyze/__init__.py
new file mode 100644
index 0000000000..457703731d
--- /dev/null
+++ b/test/lib/ansible_test/_internal/coverage/analyze/__init__.py
@@ -0,0 +1,19 @@
+"""Common logic for the `coverage analyze` subcommand."""
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from ... import types as t
+
+from .. import (
+    CoverageConfig,
+)
+
+
+class CoverageAnalyzeConfig(CoverageConfig):
+    """Configuration for the `coverage analyze` command."""
+    def __init__(self, args):  # type: (t.Any) -> None
+        super(CoverageAnalyzeConfig, self).__init__(args)
+
+        # avoid mixing log messages with file output when using `/dev/stdout` for the output file on commands
+        # this may be worth considering as the default behavior in the future, instead of being dependent on the command or options used
+        self.info_stderr = True
diff --git a/test/lib/ansible_test/_internal/coverage/analyze/targets/__init__.py b/test/lib/ansible_test/_internal/coverage/analyze/targets/__init__.py
new file mode 100644
index 0000000000..a01b804f26
--- /dev/null
+++ b/test/lib/ansible_test/_internal/coverage/analyze/targets/__init__.py
@@ -0,0 +1,115 @@
+"""Analyze integration test target code coverage."""
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+
+from .... import types as t
+
+from ....io import (
+    read_json_file,
+    write_json_file,
+)
+
+from ....util import (
+    ApplicationError,
+    display,
+)
+
+from .. import (
+    CoverageAnalyzeConfig,
+)
+
+if t.TYPE_CHECKING:
+    Arcs = t.Dict[str, t.Dict[t.Tuple[int, int], t.Set[int]]]
+    Lines = t.Dict[str, t.Dict[int, t.Set[int]]]
+    TargetIndexes = t.Dict[str, int]
+    TargetSetIndexes = t.Dict[t.FrozenSet[int], int]
+
+
+def make_report(target_indexes, arcs, lines):  # type: (TargetIndexes, Arcs, Lines) -> t.Dict[str, t.Any]
+    """Condense target indexes, arcs and lines into a compact report."""
+    set_indexes = {}
+    arc_refs = dict((path, dict((format_arc(arc), get_target_set_index(indexes, set_indexes)) for arc, indexes in data.items())) for path, data in arcs.items())
+    line_refs = dict((path, dict((line, get_target_set_index(indexes, set_indexes)) for line, indexes in data.items())) for path, data in lines.items())
+
+    report = dict(
+        targets=[name for name, index in sorted(target_indexes.items(), key=lambda kvp: kvp[1])],
+        target_sets=[sorted(data) for data, index in sorted(set_indexes.items(), key=lambda kvp: kvp[1])],
+        arcs=arc_refs,
+        lines=line_refs,
+    )
+
+    return report
+
+
+def load_report(report):  # type: (t.Dict[str, t.Any]) -> t.Tuple[t.List[str], Arcs, Lines]
+    """Extract target indexes, arcs and lines from an existing report."""
+    try:
+        target_indexes = report['targets']  # type: t.List[str]
+        target_sets = report['target_sets']  # type: t.List[t.List[int]]
+        arc_data = report['arcs']  # type: t.Dict[str, t.Dict[str, int]]
+        line_data = report['lines']  # type: t.Dict[str, t.Dict[int, int]]
+    except KeyError as ex:
+        raise ApplicationError('Document is missing key "%s".' % ex.args)
+    except TypeError:
+        raise ApplicationError('Document is type "%s" instead of "dict".' % type(report).__name__)
+
+    arcs = dict((path, dict((parse_arc(arc), set(target_sets[index])) for arc, index in data.items())) for path, data in arc_data.items())
+    lines = dict((path, dict((int(line), set(target_sets[index])) for line, index in data.items())) for path, data in line_data.items())
+
+    return target_indexes, arcs, lines
+
+
+def read_report(path):  # type: (str) -> t.Tuple[t.List[str], Arcs, Lines]
+    """Read a JSON report from disk."""
+    try:
+        report = read_json_file(path)
+    except Exception as ex:
+        raise ApplicationError('File "%s" is not valid JSON: %s' % (path, ex))
+
+    try:
+        return load_report(report)
+    except ApplicationError as ex:
+        raise ApplicationError('File "%s" is not an aggregated coverage data file. %s' % (path, ex))
+
+
+def write_report(args, report, path):  # type: (CoverageAnalyzeTargetsConfig, t.Dict[str, t.Any], str) -> None
+    """Write a JSON report to disk."""
+    if args.explain:
+        return
+
+    write_json_file(path, report, formatted=False)
+
+    display.info('Generated %d byte report with %d targets covering %d files.' % (
+        os.path.getsize(path), len(report['targets']), len(set(report['arcs'].keys()) | set(report['lines'].keys())),
+    ), verbosity=1)
+
+
+def format_arc(value):  # type: (t.Tuple[int, int]) -> str
+    """Format an arc tuple as a string."""
+    return '%d:%d' % value
+
+
+def parse_arc(value):  # type: (str) -> t.Tuple[int, int]
+    """Parse an arc string into a tuple."""
+    first, last = tuple(map(int, value.split(':')))
+    return first, last
+
+
+def get_target_set_index(data, target_set_indexes):  # type: (t.Set[int], TargetSetIndexes) -> int
+    """Find or add the target set in the result set and return the target set index."""
+    return target_set_indexes.setdefault(frozenset(data), len(target_set_indexes))
+
+
+def get_target_index(name, target_indexes):  # type: (str, TargetIndexes) -> int
+    """Find or add the target in the result set and return the target index."""
+    return target_indexes.setdefault(name, len(target_indexes))
+
+
+class CoverageAnalyzeTargetsConfig(CoverageAnalyzeConfig):
+    """Configuration for the `coverage analyze targets` command."""
+    def __init__(self, args):  # type: (t.Any) -> None
+        super(CoverageAnalyzeTargetsConfig, self).__init__(args)
+
+        self.info_stderr = True
diff --git a/test/lib/ansible_test/_internal/coverage/analyze/targets/combine.py b/test/lib/ansible_test/_internal/coverage/analyze/targets/combine.py
new file mode 100644
index 0000000000..3a98ef9a9a
--- /dev/null
+++ b/test/lib/ansible_test/_internal/coverage/analyze/targets/combine.py
@@ -0,0 +1,63 @@
+"""Combine integration test target code coverage reports."""
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from .... import types as t
+
+from . import (
+    CoverageAnalyzeTargetsConfig,
+    get_target_index,
+    make_report,
+    read_report,
+    write_report,
+)
+
+if t.TYPE_CHECKING:
+    from . import (
+        Arcs,
+        Lines,
+        TargetIndexes,
+    )
+
+
+def command_coverage_analyze_targets_combine(args):  # type: (CoverageAnalyzeTargetsCombineConfig) -> None
+    """Combine integration test target code coverage reports."""
+    combined_target_indexes = {}  # type: TargetIndexes
+    combined_path_arcs = {}  # type: Arcs
+    combined_path_lines = {}  # type: Lines
+
+    for report_path in args.input_files:
+        covered_targets, covered_path_arcs, covered_path_lines = read_report(report_path)
+
+        merge_indexes(covered_path_arcs, covered_targets, combined_path_arcs, combined_target_indexes)
+        merge_indexes(covered_path_lines, covered_targets, combined_path_lines, combined_target_indexes)
+
+    report = make_report(combined_target_indexes, combined_path_arcs, combined_path_lines)
+
+    write_report(args, report, args.output_file)
+
+
+def merge_indexes(
+        source_data,  # type: t.Dict[str, t.Dict[t.Any, t.Set[int]]]
+        source_index,  # type: t.List[str]
+        combined_data,  # type: t.Dict[str, t.Dict[t.Any, t.Set[int]]]
+        combined_index,  # type: TargetIndexes
+):  # type: (...) -> None
+    """Merge indexes from the source into the combined data set (arcs or lines)."""
+    for covered_path, covered_points in source_data.items():
+        combined_points = combined_data.setdefault(covered_path, {})
+
+        for covered_point, covered_target_indexes in covered_points.items():
+            combined_point = combined_points.setdefault(covered_point, set())
+
+            for covered_target_index in covered_target_indexes:
+                combined_point.add(get_target_index(source_index[covered_target_index], combined_index))
+
+
+class CoverageAnalyzeTargetsCombineConfig(CoverageAnalyzeTargetsConfig):
+    """Configuration for the `coverage analyze targets combine` command."""
+    def __init__(self, args):  # type: (t.Any) -> None
+        super(CoverageAnalyzeTargetsCombineConfig, self).__init__(args)
+
+        self.input_files = args.input_file  # type: t.List[str]
+        self.output_file = args.output_file  # type: str
diff --git a/test/lib/ansible_test/_internal/coverage/analyze/targets/expand.py b/test/lib/ansible_test/_internal/coverage/analyze/targets/expand.py
new file mode 100644
index 0000000000..536a877d70
--- /dev/null
+++ b/test/lib/ansible_test/_internal/coverage/analyze/targets/expand.py
@@ -0,0 +1,58 @@
+"""Expand target names in an aggregated coverage file."""
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+from .... import types as t
+
+from ....io import (
+    SortedSetEncoder,
+    write_json_file,
+)
+
+from . import (
+    CoverageAnalyzeTargetsConfig,
+    format_arc,
+    read_report,
+)
+
+
+def command_coverage_analyze_targets_expand(args):  # type: (CoverageAnalyzeTargetsExpandConfig) -> None
+    """Expand target names in an aggregated coverage file."""
+    covered_targets, covered_path_arcs, covered_path_lines = read_report(args.input_file)
+
+    report = dict(
+        arcs=expand_indexes(covered_path_arcs, covered_targets, format_arc),
+        lines=expand_indexes(covered_path_lines, covered_targets, str),
+    )
+
+    if not args.explain:
+        write_json_file(args.output_file, report, encoder=SortedSetEncoder)
+
+
+def expand_indexes(
+        source_data,  # type: t.Dict[str, t.Dict[t.Any, t.Set[int]]]
+        source_index,  # type: t.List[str]
+        format_func,  # type: t.Callable[t.Tuple[t.Any], str]
+):  # type: (...) -> t.Dict[str, t.Dict[t.Any, t.Set[str]]]
+    """Merge indexes from the source into the combined data set (arcs or lines)."""
+    combined_data = {}  # type: t.Dict[str, t.Dict[t.Any, t.Set[str]]]
+
+    for covered_path, covered_points in source_data.items():
+        combined_points = combined_data.setdefault(covered_path, {})
+
+        for covered_point, covered_target_indexes in covered_points.items():
+            combined_point = combined_points.setdefault(format_func(covered_point), set())
+
+            for covered_target_index in covered_target_indexes:
+                combined_point.add(source_index[covered_target_index])
+
+    return combined_data
+
+
+class CoverageAnalyzeTargetsExpandConfig(CoverageAnalyzeTargetsConfig):
+    """Configuration for the `coverage analyze targets expand` command."""
+    def __init__(self, args):  # type: (t.Any) -> None
+        super(CoverageAnalyzeTargetsExpandConfig, self).__init__(args)
+
+        self.input_file = args.input_file  # type: str
+        self.output_file = args.output_file  # type: str
diff --git a/test/lib/ansible_test/_internal/coverage/analyze/targets/generate.py b/test/lib/ansible_test/_internal/coverage/analyze/targets/generate.py
new file mode 100644
index 0000000000..4e33255946
--- /dev/null
+++ b/test/lib/ansible_test/_internal/coverage/analyze/targets/generate.py
@@ -0,0 +1,143 @@
+"""Analyze code coverage data to determine which integration test targets provide coverage for each arc or line."""
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+
+from .... import types as t
+
+from ....encoding import (
+    to_text,
+)
+
+from ....data import (
+    data_context,
+)
+
+from ....util_common import (
+    ResultType,
+)
+
+from ... import (
+    enumerate_powershell_lines,
+    enumerate_python_arcs,
+    get_collection_path_regexes,
+    get_powershell_coverage_files,
+    get_python_coverage_files,
+    get_python_modules,
+    initialize_coverage,
+    PathChecker,
+)
+
+from . import (
+    CoverageAnalyzeTargetsConfig,
+    get_target_index,
+    make_report,
+    write_report,
+)
+
+if t.TYPE_CHECKING:
+    from . import (
+        Arcs,
+        Lines,
+        TargetIndexes,
+    )
+
+
+def command_coverage_analyze_targets_generate(args):  # type: (CoverageAnalyzeTargetsGenerateConfig) -> None
+    """Analyze code coverage data to determine which integration test targets provide coverage for each arc or line."""
+    root = data_context().content.root
+    target_indexes = {}
+    arcs = dict((os.path.relpath(path, root), data) for path, data in analyze_python_coverage(args, target_indexes).items())
+    lines = dict((os.path.relpath(path, root), data) for path, data in analyze_powershell_coverage(args, target_indexes).items())
+    report = make_report(target_indexes, arcs, lines)
+    write_report(args, report, args.output_file)
+
+
+def analyze_python_coverage(
+        args,  # type: CoverageAnalyzeTargetsConfig
+        target_indexes,  # type: TargetIndexes
+):  # type: (...) -> Arcs
+    """Analyze Python code coverage."""
+    results = {}  # type: Arcs
+    collection_search_re, collection_sub_re = get_collection_path_regexes()
+    modules = get_python_modules()
+    python_files = get_python_coverage_files()
+    coverage = initialize_coverage(args)
+
+    for python_file in python_files:
+        if not is_integration_coverage_file(python_file):
+            continue
+
+        target_name = get_target_name(python_file)
+        target_index = get_target_index(target_name, target_indexes)
+
+        for filename, covered_arcs in enumerate_python_arcs(python_file, coverage, modules, collection_search_re, collection_sub_re):
+            arcs = results.setdefault(filename, {})
+
+            for covered_arc in covered_arcs:
+                arc = arcs.setdefault(covered_arc, set())
+                arc.add(target_index)
+
+    prune_invalid_filenames(args, results, collection_search_re=collection_search_re)
+
+    return results
+
+
+def analyze_powershell_coverage(
+        args,  # type: CoverageAnalyzeTargetsConfig
+        target_indexes,  # type: TargetIndexes
+):  # type: (...) -> Lines
+    """Analyze PowerShell code coverage"""
+    results = {}  # type: Lines
+    powershell_files = get_powershell_coverage_files()
+
+    for powershell_file in powershell_files:
+        if not is_integration_coverage_file(powershell_file):
+            continue
+
+        target_name = get_target_name(powershell_file)
+        target_index = get_target_index(target_name, target_indexes)
+
+        for filename, hits in enumerate_powershell_lines(powershell_file):
+            lines = results.setdefault(filename, {})
+
+            for covered_line in hits:
+                line = lines.setdefault(covered_line, set())
+                line.add(target_index)
+
+    prune_invalid_filenames(args, results)
+
+    return results
+
+
+def prune_invalid_filenames(
+        args,  # type: CoverageAnalyzeTargetsConfig
+        results,  # type: t.Dict[str, t.Any]
+        collection_search_re=None,  # type: t.Optional[str]
+):  # type: (...) -> None
+    """Remove invalid filenames from the given result set."""
+    path_checker = PathChecker(args, collection_search_re)
+
+    for path in list(results.keys()):
+        if not path_checker.check_path(path):
+            del results[path]
+
+
+def get_target_name(path):  # type: (str) -> str
+    """Extract the test target name from the given coverage path."""
+    return to_text(os.path.basename(path).split('=')[1])
+
+
+def is_integration_coverage_file(path):  # type: (str) -> bool
+    """Returns True if the coverage file came from integration tests, otherwise False."""
+    return os.path.basename(path).split('=')[0] in ('integration', 'windows-integration', 'network-integration')
+
+
+class CoverageAnalyzeTargetsGenerateConfig(CoverageAnalyzeTargetsConfig):
+    """Configuration for the `coverage analyze targets generate` command."""
+    def __init__(self, args):  # type: (t.Any) -> None
+        super(CoverageAnalyzeTargetsGenerateConfig, self).__init__(args)
+
+        self.input_dir = args.input_dir or ResultType.COVERAGE.path  # type: str
+        self.output_file = args.output_file  # type: str
diff --git a/test/lib/ansible_test/_internal/coverage/analyze/targets/missing.py b/test/lib/ansible_test/_internal/coverage/analyze/targets/missing.py
new file mode 100644
index 0000000000..22443615eb
--- /dev/null
+++ b/test/lib/ansible_test/_internal/coverage/analyze/targets/missing.py
@@ -0,0 +1,110 @@
+"""Identify aggregated coverage in one file missing from another."""
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import os
+
+from .... import types as t
+
+from ....encoding import (
+    to_bytes,
+)
+
+from . import (
+    CoverageAnalyzeTargetsConfig,
+    get_target_index,
+    make_report,
+    read_report,
+    write_report,
+)
+
+if t.TYPE_CHECKING:
+    from . import (
+        TargetIndexes,
+    )
+
+    TargetKey = t.TypeVar('TargetKey', int, t.Tuple[int, int])
+
+
+def command_coverage_analyze_targets_missing(args):  # type: (CoverageAnalyzeTargetsMissingConfig) -> None
+    """Identify aggregated coverage in one file missing from another."""
+    from_targets, from_path_arcs, from_path_lines = read_report(args.from_file)
+    to_targets, to_path_arcs, to_path_lines = read_report(args.to_file)
+    target_indexes = {}
+
+    if args.only_gaps:
+        arcs = find_gaps(from_path_arcs, from_targets, to_path_arcs, target_indexes, args.only_exists)
+        lines = find_gaps(from_path_lines, from_targets, to_path_lines, target_indexes, args.only_exists)
+    else:
+        arcs = find_missing(from_path_arcs, from_targets, to_path_arcs, to_targets, target_indexes, args.only_exists)
+        lines = find_missing(from_path_lines, from_targets, to_path_lines, to_targets, target_indexes, args.only_exists)
+
+    report = make_report(target_indexes, arcs, lines)
+    write_report(args, report, args.output_file)
+
+
+def find_gaps(
+        from_data,  # type: t.Dict[str, t.Dict[TargetKey, t.Set[int]]]
+        from_index,  # type: t.List[str]
+        to_data,  # type: t.Dict[str, t.Dict[TargetKey, t.Set[int]]]
+        target_indexes,  # type: TargetIndexes,
+        only_exists,  # type: bool
+):  # type: (...) -> t.Dict[str, t.Dict[TargetKey, t.Set[int]]]
+    """Find gaps in coverage between the from and to data sets."""
+    target_data = {}
+
+    for from_path, from_points in from_data.items():
+        if only_exists and not os.path.isfile(to_bytes(from_path)):
+            continue
+
+        to_points = to_data.get(from_path, {})
+
+        gaps = set(from_points.keys()) - set(to_points.keys())
+
+        if gaps:
+            gap_points = dict((key, value) for key, value in from_points.items() if key in gaps)
+            target_data[from_path] = dict((gap, set(get_target_index(from_index[i], target_indexes) for i in indexes)) for gap, indexes in gap_points.items())
+
+    return target_data
+
+
+def find_missing(
+        from_data,  # type: t.Dict[str, t.Dict[TargetKey, t.Set[int]]]
+        from_index,  # type: t.List[str]
+        to_data,  # type: t.Dict[str, t.Dict[TargetKey, t.Set[int]]]
+        to_index,  # type: t.List[str]
+        target_indexes,  # type: TargetIndexes,
+        only_exists,  # type: bool
+):  # type: (...) -> t.Dict[str, t.Dict[TargetKey, t.Set[int]]]
+    """Find coverage in from_data not present in to_data (arcs or lines)."""
+    target_data = {}
+
+    for from_path, from_points in from_data.items():
+        if only_exists and not os.path.isfile(to_bytes(from_path)):
+            continue
+
+        to_points = to_data.get(from_path, {})
+
+        for from_point, from_target_indexes in from_points.items():
+            to_target_indexes = to_points.get(from_point, set())
+
+            remaining_targets = set(from_index[i] for i in from_target_indexes) - set(to_index[i] for i in to_target_indexes)
+
+            if remaining_targets:
+                target_index = target_data.setdefault(from_path, {}).setdefault(from_point, set())
+                target_index.update(get_target_index(name, target_indexes) for name in remaining_targets)
+
+    return target_data
+
+
+class CoverageAnalyzeTargetsMissingConfig(CoverageAnalyzeTargetsConfig):
+    """Configuration for the `coverage analyze targets missing` command."""
+    def __init__(self, args):  # type: (t.Any) -> None
+        super(CoverageAnalyzeTargetsMissingConfig, self).__init__(args)
+
+        self.from_file = args.from_file  # type: str
+        self.to_file = args.to_file  # type: str
+        self.output_file = args.output_file  # type: str
+
+        self.only_gaps = args.only_gaps  # type: bool
+        self.only_exists = args.only_exists  # type: bool
diff --git a/test/lib/ansible_test/_internal/coverage/combine.py b/test/lib/ansible_test/_internal/coverage/combine.py
index a07a4dd6de..e4a6f61415 100644
--- a/test/lib/ansible_test/_internal/coverage/combine.py
+++ b/test/lib/ansible_test/_internal/coverage/combine.py
@@ -3,16 +3,13 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import os
-import re
 
 from ..target import (
-    walk_module_targets,
     walk_compile_targets,
     walk_powershell_targets,
 )
 
 from ..io import (
-    read_json_file,
     read_text_file,
 )
 
@@ -25,15 +22,18 @@ from ..util_common import (
     write_json_test_results,
 )
 
-from ..data import (
-    data_context,
-)
-
 from . import (
+    enumerate_python_arcs,
+    enumerate_powershell_lines,
+    get_collection_path_regexes,
+    get_python_coverage_files,
+    get_python_modules,
+    get_powershell_coverage_files,
     initialize_coverage,
     COVERAGE_OUTPUT_FILE_NAME,
     COVERAGE_GROUPS,
     CoverageConfig,
+    PathChecker,
 )
 
 
@@ -57,58 +57,27 @@ def _command_coverage_combine_python(args):
     """
     coverage = initialize_coverage(args)
 
-    modules = dict((target.module, target.path) for target in list(walk_module_targets()) if target.path.endswith('.py'))
+    modules = get_python_modules()
 
-    coverage_dir = ResultType.COVERAGE.path
-    coverage_files = [os.path.join(coverage_dir, f) for f in os.listdir(coverage_dir)
-                      if '=coverage.' in f and '=python' in f]
+    coverage_files = get_python_coverage_files()
 
     counter = 0
     sources = _get_coverage_targets(args, walk_compile_targets)
     groups = _build_stub_groups(args, sources, lambda line_count: set())
 
-    if data_context().content.collection:
-        collection_search_re = re.compile(r'/%s/' % data_context().content.collection.directory)
-        collection_sub_re = re.compile(r'^.*?/%s/' % data_context().content.collection.directory)
-    else:
-        collection_search_re = None
-        collection_sub_re = None
+    collection_search_re, collection_sub_re = get_collection_path_regexes()
 
     for coverage_file in coverage_files:
         counter += 1
         display.info('[%4d/%4d] %s' % (counter, len(coverage_files), coverage_file), verbosity=2)
 
-        original = coverage.CoverageData()
-
         group = get_coverage_group(args, coverage_file)
 
         if group is None:
             display.warning('Unexpected name for coverage file: %s' % coverage_file)
             continue
 
-        if os.path.getsize(coverage_file) == 0:
-            display.warning('Empty coverage file: %s' % coverage_file)
-            continue
-
-        try:
-            original.read_file(coverage_file)
-        except Exception as ex:  # pylint: disable=locally-disabled, broad-except
-            display.error(u'%s' % ex)
-            continue
-
-        for filename in original.measured_files():
-            arcs = set(original.arcs(filename) or [])
-
-            if not arcs:
-                # This is most likely due to using an unsupported version of coverage.
-                display.warning('No arcs found for "%s" in coverage file: %s' % (filename, coverage_file))
-                continue
-
-            filename = _sanitize_filename(filename, modules=modules, collection_search_re=collection_search_re,
-                                          collection_sub_re=collection_sub_re)
-            if not filename:
-                continue
-
+        for filename, arcs in enumerate_python_arcs(coverage_file, coverage, modules, collection_search_re, collection_sub_re):
             if group not in groups:
                 groups[group] = {}
 
@@ -120,28 +89,18 @@ def _command_coverage_combine_python(args):
             arc_data[filename].update(arcs)
 
     output_files = []
-    invalid_path_count = 0
-    invalid_path_chars = 0
 
     coverage_file = os.path.join(ResultType.COVERAGE.path, COVERAGE_OUTPUT_FILE_NAME)
 
+    path_checker = PathChecker(args, collection_search_re)
+
     for group in sorted(groups):
         arc_data = groups[group]
 
         updated = coverage.CoverageData()
 
         for filename in arc_data:
-            if not os.path.isfile(filename):
-                if collection_search_re and collection_search_re.search(filename) and os.path.basename(filename) == '__init__.py':
-                    # the collection loader uses implicit namespace packages, so __init__.py does not need to exist on disk
-                    continue
-
-                invalid_path_count += 1
-                invalid_path_chars += len(filename)
-
-                if args.verbosity > 1:
-                    display.warning('Invalid coverage path: %s' % filename)
-
+            if not path_checker.check_path(filename):
                 continue
 
             updated.add_arcs({filename: list(arc_data[filename])})
@@ -154,8 +113,7 @@ def _command_coverage_combine_python(args):
             updated.write_file(output_file)
             output_files.append(output_file)
 
-    if invalid_path_count > 0:
-        display.warning('Ignored %d characters from %d invalid coverage path(s).' % (invalid_path_chars, invalid_path_count))
+    path_checker.report()
 
     return sorted(output_files)
 
@@ -165,9 +123,7 @@ def _command_coverage_combine_powershell(args):
     :type args: CoverageConfig
     :rtype: list[str]
     """
-    coverage_dir = ResultType.COVERAGE.path
-    coverage_files = [os.path.join(coverage_dir, f) for f in os.listdir(coverage_dir)
-                      if '=coverage.' in f and '=powershell' in f]
+    coverage_files = get_powershell_coverage_files()
 
     def _default_stub_value(lines):
         val = {}
@@ -189,57 +145,26 @@ def _command_coverage_combine_powershell(args):
             display.warning('Unexpected name for coverage file: %s' % coverage_file)
             continue
 
-        if os.path.getsize(coverage_file) == 0:
-            display.warning('Empty coverage file: %s' % coverage_file)
-            continue
-
-        try:
-            coverage_run = read_json_file(coverage_file)
-        except Exception as ex:  # pylint: disable=locally-disabled, broad-except
-            display.error(u'%s' % ex)
-            continue
-
-        for filename, hit_info in coverage_run.items():
+        for filename, hits in enumerate_powershell_lines(coverage_file):
             if group not in groups:
                 groups[group] = {}
 
             coverage_data = groups[group]
 
-            filename = _sanitize_filename(filename)
-            if not filename:
-                continue
-
             if filename not in coverage_data:
                 coverage_data[filename] = {}
 
             file_coverage = coverage_data[filename]
 
-            if not isinstance(hit_info, list):
-                hit_info = [hit_info]
-
-            for hit_entry in hit_info:
-                if not hit_entry:
-                    continue
-
-                line_count = file_coverage.get(hit_entry['Line'], 0) + hit_entry['HitCount']
-                file_coverage[hit_entry['Line']] = line_count
+            for line_no, hit_count in hits.items():
+                file_coverage[line_no] = file_coverage.get(line_no, 0) + hit_count
 
     output_files = []
-    invalid_path_count = 0
-    invalid_path_chars = 0
 
-    for group in sorted(groups):
-        coverage_data = groups[group]
+    path_checker = PathChecker(args)
 
-        for filename in coverage_data:
-            if not os.path.isfile(filename):
-                invalid_path_count += 1
-                invalid_path_chars += len(filename)
-
-                if args.verbosity > 1:
-                    display.warning('Invalid coverage path: %s' % filename)
-
-                continue
+    for group in sorted(groups):
+        coverage_data = dict((filename, data) for filename, data in groups[group].items() if path_checker.check_path(filename))
 
         if args.all:
             # Add 0 line entries for files not in coverage_data
@@ -256,9 +181,7 @@ def _command_coverage_combine_powershell(args):
 
             output_files.append(os.path.join(ResultType.COVERAGE.path, output_file))
 
-    if invalid_path_count > 0:
-        display.warning(
-            'Ignored %d characters from %d invalid coverage path(s).' % (invalid_path_chars, invalid_path_count))
+    path_checker.report()
 
     return sorted(output_files)
 
@@ -346,67 +269,3 @@ def get_coverage_group(args, coverage_file):
             group += '=%s' % names[part]
 
     return group
-
-
-def _sanitize_filename(filename, modules=None, collection_search_re=None, collection_sub_re=None):
-    """
-    :type filename: str
-    :type modules: dict | None
-    :type collection_search_re: Pattern | None
-    :type collection_sub_re: Pattern | None
-    :rtype: str | None
-    """
-    ansible_path = os.path.abspath('lib/ansible/') + '/'
-    root_path = data_context().content.root + '/'
-    integration_temp_path = os.path.sep + os.path.join(ResultType.TMP.relative_path, 'integration') + os.path.sep
-
-    if modules is None:
-        modules = {}
-
-    if '/ansible_modlib.zip/ansible/' in filename:
-        # Rewrite the module_utils path from the remote host to match the controller. Ansible 2.6 and earlier.
-        new_name = re.sub('^.*/ansible_modlib.zip/ansible/', ansible_path, filename)
-        display.info('%s -> %s' % (filename, new_name), verbosity=3)
-        filename = new_name
-    elif collection_search_re and collection_search_re.search(filename):
-        new_name = os.path.abspath(collection_sub_re.sub('', filename))
-        display.info('%s -> %s' % (filename, new_name), verbosity=3)
-        filename = new_name
-    elif re.search(r'/ansible_[^/]+_payload\.zip/ansible/', filename):
-        # Rewrite the module_utils path from the remote host to match the controller. Ansible 2.7 and later.
-        new_name = re.sub(r'^.*/ansible_[^/]+_payload\.zip/ansible/', ansible_path, filename)
-        display.info('%s -> %s' % (filename, new_name), verbosity=3)
-        filename = new_name
-    elif '/ansible_module_' in filename:
-        # Rewrite the module path from the remote host to match the controller. Ansible 2.6 and earlier.
-        module_name = re.sub('^.*/ansible_module_(?P<module>.*).py$', '\\g<module>', filename)
-        if module_name not in modules:
-            display.warning('Skipping coverage of unknown module: %s' % module_name)
-            return None
-        new_name = os.path.abspath(modules[module_name])
-        display.info('%s -> %s' % (filename, new_name), verbosity=3)
-        filename = new_name
-    elif re.search(r'/ansible_[^/]+_payload(_[^/]+|\.zip)/__main__\.py$', filename):
-        # Rewrite the module path from the remote host to match the controller. Ansible 2.7 and later.
-        # AnsiballZ versions using zipimporter will match the `.zip` portion of the regex.
-        # AnsiballZ versions not using zipimporter will match the `_[^/]+` portion of the regex.
-        module_name = re.sub(r'^.*/ansible_(?P<module>[^/]+)_payload(_[^/]+|\.zip)/__main__\.py$',
-                             '\\g<module>', filename).rstrip('_')
-        if module_name not in modules:
-            display.warning('Skipping coverage of unknown module: %s' % module_name)
-            return None
-        new_name = os.path.abspath(modules[module_name])
-        display.info('%s -> %s' % (filename, new_name), verbosity=3)
-        filename = new_name
-    elif re.search('^(/.*?)?/root/ansible/', filename):
-        # Rewrite the path of code running on a remote host or in a docker container as root.
-        new_name = re.sub('^(/.*?)?/root/ansible/', root_path, filename)
-        display.info('%s -> %s' % (filename, new_name), verbosity=3)
-        filename = new_name
-    elif integration_temp_path in filename:
-        # Rewrite the path of code running from an integration test temporary directory.
-        new_name = re.sub(r'^.*' + re.escape(integration_temp_path) + '[^/]+/', root_path, filename)
-        display.info('%s -> %s' % (filename, new_name), verbosity=3)
-        filename = new_name
-
-    return filename
diff --git a/test/lib/ansible_test/_internal/coverage/erase.py b/test/lib/ansible_test/_internal/coverage/erase.py
index 8b1f6f3b11..92d241c7fb 100644
--- a/test/lib/ansible_test/_internal/coverage/erase.py
+++ b/test/lib/ansible_test/_internal/coverage/erase.py
@@ -9,17 +9,12 @@ from ..util_common import (
 )
 
 from . import (
-    initialize_coverage,
     CoverageConfig,
 )
 
 
-def command_coverage_erase(args):
-    """
-    :type args: CoverageConfig
-    """
-    initialize_coverage(args)
-
+def command_coverage_erase(args):  # type: (CoverageConfig) -> None
+    """Erase code coverage data files collected during test runs."""
     coverage_dir = ResultType.COVERAGE.path
 
     for name in os.listdir(coverage_dir):
diff --git a/test/lib/ansible_test/_internal/io.py b/test/lib/ansible_test/_internal/io.py
index 8daa4dda01..0f61cd2df2 100644
--- a/test/lib/ansible_test/_internal/io.py
+++ b/test/lib/ansible_test/_internal/io.py
@@ -41,9 +41,20 @@ def make_dirs(path):  # type: (str) -> None
             raise
 
 
-def write_json_file(path, content, create_directories=False, formatted=True):  # type: (str, t.Union[t.List[t.Any], t.Dict[str, t.Any]], bool, bool) -> None
+def write_json_file(path,  # type: str
+                    content,  # type: t.Union[t.List[t.Any], t.Dict[str, t.Any]]
+                    create_directories=False,  # type: bool
+                    formatted=True,  # type: bool
+                    encoder=None,  # type: t.Optional[t.Callable[[t.Any], t.Any]]
+                    ):  # type: (...) -> None
     """Write the given json content to the specified path, optionally creating missing directories."""
-    text_content = json.dumps(content, sort_keys=formatted, indent=4 if formatted else None, separators=(', ', ': ') if formatted else (',', ':')) + '\n'
+    text_content = json.dumps(content,
+                              sort_keys=formatted,
+                              indent=4 if formatted else None,
+                              separators=(', ', ': ') if formatted else (',', ':'),
+                              cls=encoder,
+                              ) + '\n'
+
     write_text_file(path, text_content, create_directories=create_directories)
 
 
@@ -72,3 +83,12 @@ def open_binary_file(path, mode='rb'):  # type: (str, str) -> t.BinaryIO
 
     # noinspection PyTypeChecker
     return io.open(to_bytes(path), mode)
+
+
+class SortedSetEncoder(json.JSONEncoder):
+    """Encode sets as sorted lists."""
+    def default(self, obj):  # pylint: disable=method-hidden, arguments-differ
+        if isinstance(obj, set):
+            return sorted(obj)
+
+        return super(SortedSetEncoder).default(self, obj)
diff --git a/test/lib/ansible_test/_internal/util_common.py b/test/lib/ansible_test/_internal/util_common.py
index c7ff422752..1cb0602358 100644
--- a/test/lib/ansible_test/_internal/util_common.py
+++ b/test/lib/ansible_test/_internal/util_common.py
@@ -101,6 +101,8 @@ class CommonConfig:
         self.truncate = args.truncate  # type: int
         self.redact = args.redact  # type: bool
 
+        self.info_stderr = False  # type: bool
+
         self.cache = {}
 
     def get_ansible_config(self):  # type: () -> str
@@ -143,10 +145,15 @@ def named_temporary_file(args, prefix, suffix, directory, content):
             yield tempfile_fd.name
 
 
-def write_json_test_results(category, name, content, formatted=True):  # type: (ResultType, str, t.Union[t.List[t.Any], t.Dict[str, t.Any]], bool) -> None
+def write_json_test_results(category,  # type: ResultType
+                            name,  # type: str
+                            content,  # type: t.Union[t.List[t.Any], t.Dict[str, t.Any]]
+                            formatted=True,  # type: bool
+                            encoder=None,  # type: t.Optional[t.Callable[[t.Any], t.Any]]
+                            ):  # type: (...) -> None
     """Write the given json content to the specified test results path, creating directories as needed."""
     path = os.path.join(category.path, name)
-    write_json_file(path, content, create_directories=True, formatted=formatted)
+    write_json_file(path, content, create_directories=True, formatted=formatted, encoder=encoder)
 
 
 def write_text_test_results(category, name, content):  # type: (ResultType, str, str) -> None
diff --git a/test/utils/shippable/shippable.sh b/test/utils/shippable/shippable.sh
index 52d62a5d1c..4141eee87f 100755
--- a/test/utils/shippable/shippable.sh
+++ b/test/utils/shippable/shippable.sh
@@ -103,6 +103,9 @@ function cleanup
             ansible-test coverage xml --color -v --requirements --group-by command --group-by version ${stub:+"$stub"}
             cp -a test/results/reports/coverage=*.xml shippable/codecoverage/
 
+            # analyze and capture code coverage aggregated by integration test target
+            ansible-test coverage analyze targets generate -v shippable/testresults/coverage-analyze-targets.json
+
             # upload coverage report to codecov.io only when using complete on-demand coverage
             if [ "${COVERAGE}" == "--coverage" ] && [ "${CHANGED}" == "" ]; then
                 for file in test/results/reports/coverage=*.xml; do