Prepare ansible-test for supporting collections. (#58886)

This is a small but incomplete set of the initial changes for supporting testing of collections with ansible-test.

17 files changed, 130 insertions, 59 deletions

diff --git a/test/runner/lib/ansible_util.py b/test/runner/lib/ansible_util.py
index f7c2abb464..86cb7927ad 100644
--- a/test/runner/lib/ansible_util.py
+++ b/test/runner/lib/ansible_util.py
@@ -15,6 +15,7 @@ from lib.util import (
     find_python,
     run_command,
     ApplicationError,
+    INSTALL_ROOT,
 )
 
 from lib.config import (
@@ -36,7 +37,7 @@ def ansible_environment(args, color=True, ansible_config=None):
     env = common_environment()
     path = env['PATH']
 
-    ansible_path = os.path.join(os.getcwd(), 'bin')
+    ansible_path = os.path.join(INSTALL_ROOT, 'bin')
 
     if not path.startswith(ansible_path + os.path.pathsep):
         path = ansible_path + os.path.pathsep + path
@@ -44,9 +45,9 @@ def ansible_environment(args, color=True, ansible_config=None):
     if ansible_config:
         pass
     elif isinstance(args, IntegrationConfig):
-        ansible_config = 'test/integration/%s.cfg' % args.command
+        ansible_config = os.path.join(INSTALL_ROOT, 'test/integration/%s.cfg' % args.command)
     else:
-        ansible_config = 'test/%s/ansible.cfg' % args.command
+        ansible_config = os.path.join(INSTALL_ROOT, 'test/%s/ansible.cfg' % args.command)
 
     if not args.explain and not os.path.exists(ansible_config):
         raise ApplicationError('Configuration not found: %s' % ansible_config)
@@ -59,7 +60,7 @@ def ansible_environment(args, color=True, ansible_config=None):
         ANSIBLE_RETRY_FILES_ENABLED='false',
         ANSIBLE_CONFIG=os.path.abspath(ansible_config),
         ANSIBLE_LIBRARY='/dev/null',
-        PYTHONPATH=os.path.abspath('lib'),
+        PYTHONPATH=os.path.join(INSTALL_ROOT, 'lib'),
         PAGER='/bin/cat',
         PATH=path,
     )
@@ -84,7 +85,7 @@ def check_pyyaml(args, version):
         return
 
     python = find_python(version)
-    stdout, _dummy = run_command(args, [python, 'test/runner/yamlcheck.py'], capture=True)
+    stdout, _dummy = run_command(args, [python, os.path.join(INSTALL_ROOT, 'test/runner/yamlcheck.py')], capture=True)
 
     if args.explain:
         return
diff --git a/test/runner/lib/config.py b/test/runner/lib/config.py
index e3cffe7fb3..7292211179 100644
--- a/test/runner/lib/config.py
+++ b/test/runner/lib/config.py
@@ -5,6 +5,8 @@ from __future__ import absolute_import, print_function
 import os
 import sys
 
+import lib.types as t
+
 from lib.util import (
     CommonConfig,
     is_shippable,
@@ -112,6 +114,7 @@ class TestConfig(EnvironmentConfig):
         self.coverage = args.coverage  # type: bool
         self.coverage_label = args.coverage_label  # type: str
         self.coverage_check = args.coverage_check  # type: bool
+        self.coverage_config_base_path = None  # type: t.Optional[str]
         self.include = args.include or []  # type: list [str]
         self.exclude = args.exclude or []  # type: list [str]
         self.require = args.require or []  # type: list [str]
diff --git a/test/runner/lib/executor.py b/test/runner/lib/executor.py
index 70359cb54e..63bffe0277 100644
--- a/test/runner/lib/executor.py
+++ b/test/runner/lib/executor.py
@@ -61,6 +61,7 @@ from lib.util import (
     named_temporary_file,
     COVERAGE_OUTPUT_PATH,
     cmd_quote,
+    INSTALL_ROOT,
 )
 
 from lib.docker_util import (
@@ -273,10 +274,10 @@ def generate_egg_info(args):
     """
     :type args: EnvironmentConfig
     """
-    if os.path.isdir('lib/ansible.egg-info'):
+    if os.path.isdir(os.path.join(INSTALL_ROOT, 'lib/ansible.egg-info')):
         return
 
-    run_command(args, [args.python_executable, 'setup.py', 'egg_info'], capture=args.verbosity < 3)
+    run_command(args, [args.python_executable, 'setup.py', 'egg_info'], cwd=INSTALL_ROOT, capture=args.verbosity < 3)
 
 
 def generate_pip_install(pip, command, packages=None):
@@ -1796,9 +1797,10 @@ class EnvironmentDescription(object):
         versions += SUPPORTED_PYTHON_VERSIONS
         versions += list(set(v.split('.')[0] for v in SUPPORTED_PYTHON_VERSIONS))
 
+        version_check = os.path.join(INSTALL_ROOT, 'test/runner/versions.py')
         python_paths = dict((v, find_executable('python%s' % v, required=False)) for v in sorted(versions))
         pip_paths = dict((v, find_executable('pip%s' % v, required=False)) for v in sorted(versions))
-        program_versions = dict((v, self.get_version([python_paths[v], 'test/runner/versions.py'], warnings)) for v in sorted(python_paths) if python_paths[v])
+        program_versions = dict((v, self.get_version([python_paths[v], version_check], warnings)) for v in sorted(python_paths) if python_paths[v])
         pip_interpreters = dict((v, self.get_shebang(pip_paths[v])) for v in sorted(pip_paths) if pip_paths[v])
         known_hosts_hash = self.get_hash(os.path.expanduser('~/.ssh/known_hosts'))
 
diff --git a/test/runner/lib/git.py b/test/runner/lib/git.py
index f195f95963..20ac5bc8d1 100644
--- a/test/runner/lib/git.py
+++ b/test/runner/lib/git.py
@@ -2,13 +2,7 @@
 
 from __future__ import absolute_import, print_function
 
-try:
-    # noinspection PyUnresolvedReferences
-    from typing import (
-        Optional,
-    )
-except ImportError:
-    pass
+import lib.types as t
 
 from lib.util import (
     SubprocessError,
@@ -18,7 +12,7 @@ from lib.util import (
 
 class Git(object):
     """Wrapper around git command-line tools."""
-    def __init__(self, root=None):  # type: (Optional[str]) -> None
+    def __init__(self, root=None):  # type: (t.Optional[str]) -> None
         self.git = 'git'
         self.root = root
 
diff --git a/test/runner/lib/integration/__init__.py b/test/runner/lib/integration/__init__.py
index c3decaa571..c6f82b1118 100644
--- a/test/runner/lib/integration/__init__.py
+++ b/test/runner/lib/integration/__init__.py
@@ -6,7 +6,6 @@ import contextlib
 import json
 import os
 import shutil
-import stat
 import tempfile
 
 from lib.target import (
@@ -30,6 +29,7 @@ from lib.util import (
     MODE_DIRECTORY,
     MODE_DIRECTORY_WRITE,
     MODE_FILE,
+    INSTALL_ROOT,
 )
 
 from lib.cache import (
@@ -172,9 +172,9 @@ def integration_test_environment(args, target, inventory_path):
         ansible_config = os.path.join(integration_dir, '%s.cfg' % args.command)
 
         file_copies = [
-            ('test/integration/%s.cfg' % args.command, ansible_config),
-            ('test/integration/integration_config.yml', os.path.join(integration_dir, vars_file)),
-            (inventory_path, os.path.join(integration_dir, inventory_name)),
+            (os.path.join(INSTALL_ROOT, 'test/integration/%s.cfg' % args.command), ansible_config),
+            (os.path.join(INSTALL_ROOT, 'test/integration/integration_config.yml'), os.path.join(integration_dir, vars_file)),
+            (os.path.join(INSTALL_ROOT, inventory_path), os.path.join(integration_dir, inventory_name)),
         ]
 
         file_copies += [(path, os.path.join(temp_dir, path)) for path in files_needed]
diff --git a/test/runner/lib/sanity/ansible_doc.py b/test/runner/lib/sanity/ansible_doc.py
index a4fa66726d..cfbe2e5549 100644
--- a/test/runner/lib/sanity/ansible_doc.py
+++ b/test/runner/lib/sanity/ansible_doc.py
@@ -39,7 +39,7 @@ class AnsibleDocTest(SanityMultipleVersion):
         :rtype: TestResult
         """
         skip_file = 'test/sanity/ansible-doc/skip.txt'
-        skip_modules = set(read_lines_without_comments(skip_file, remove_blank_lines=True))
+        skip_modules = set(read_lines_without_comments(skip_file, remove_blank_lines=True, optional=True))
 
         # This should use documentable plugins from constants instead
         plugin_type_blacklist = set([
diff --git a/test/runner/lib/sanity/import.py b/test/runner/lib/sanity/import.py
index 943da2ee00..5c2c0a477a 100644
--- a/test/runner/lib/sanity/import.py
+++ b/test/runner/lib/sanity/import.py
@@ -46,7 +46,8 @@ class ImportTest(SanityMultipleVersion):
         :rtype: TestResult
         """
         skip_file = 'test/sanity/import/skip.txt'
-        skip_paths = read_lines_without_comments(skip_file, remove_blank_lines=True)
+        skip_paths = read_lines_without_comments(skip_file, remove_blank_lines=True, optional=True)
+
         skip_paths_set = set(skip_paths)
 
         paths = sorted(
diff --git a/test/runner/lib/sanity/pep8.py b/test/runner/lib/sanity/pep8.py
index 0cec3d6b59..bba262bf20 100644
--- a/test/runner/lib/sanity/pep8.py
+++ b/test/runner/lib/sanity/pep8.py
@@ -17,6 +17,7 @@ from lib.util import (
     run_command,
     read_lines_without_comments,
     parse_to_list_of_dict,
+    INSTALL_ROOT,
 )
 
 from lib.config import (
@@ -39,13 +40,13 @@ class Pep8Test(SanitySingleVersion):
         :type targets: SanityTargets
         :rtype: TestResult
         """
-        skip_paths = read_lines_without_comments(PEP8_SKIP_PATH)
-        legacy_paths = read_lines_without_comments(PEP8_LEGACY_PATH)
+        skip_paths = read_lines_without_comments(PEP8_SKIP_PATH, optional=True)
+        legacy_paths = read_lines_without_comments(PEP8_LEGACY_PATH, optional=True)
 
-        legacy_ignore_file = 'test/sanity/pep8/legacy-ignore.txt'
+        legacy_ignore_file = os.path.join(INSTALL_ROOT, 'test/sanity/pep8/legacy-ignore.txt')
         legacy_ignore = set(read_lines_without_comments(legacy_ignore_file, remove_blank_lines=True))
 
-        current_ignore_file = 'test/sanity/pep8/current-ignore.txt'
+        current_ignore_file = os.path.join(INSTALL_ROOT, 'test/sanity/pep8/current-ignore.txt')
         current_ignore = sorted(read_lines_without_comments(current_ignore_file, remove_blank_lines=True))
 
         skip_paths_set = set(skip_paths)
diff --git a/test/runner/lib/sanity/pslint.py b/test/runner/lib/sanity/pslint.py
index 5fa2d46fb0..6e629dd2d8 100644
--- a/test/runner/lib/sanity/pslint.py
+++ b/test/runner/lib/sanity/pslint.py
@@ -42,11 +42,11 @@ class PslintTest(SanitySingleVersion):
         :type targets: SanityTargets
         :rtype: TestResult
         """
-        skip_paths = read_lines_without_comments(PSLINT_SKIP_PATH)
+        skip_paths = read_lines_without_comments(PSLINT_SKIP_PATH, optional=True)
 
         invalid_ignores = []
 
-        ignore_entries = read_lines_without_comments(PSLINT_IGNORE_PATH)
+        ignore_entries = read_lines_without_comments(PSLINT_IGNORE_PATH, optional=True)
         ignore = collections.defaultdict(dict)
         line = 0
 
diff --git a/test/runner/lib/sanity/rstcheck.py b/test/runner/lib/sanity/rstcheck.py
index a6a4226cf1..8dd6a5d828 100644
--- a/test/runner/lib/sanity/rstcheck.py
+++ b/test/runner/lib/sanity/rstcheck.py
@@ -17,6 +17,7 @@ from lib.util import (
     parse_to_list_of_dict,
     display,
     read_lines_without_comments,
+    INSTALL_ROOT,
 )
 
 from lib.config import (
@@ -40,7 +41,7 @@ class RstcheckTest(SanitySingleVersion):
             display.warning('Skipping rstcheck on unsupported Python version %s.' % args.python_version)
             return SanitySkipped(self.name)
 
-        ignore_file = 'test/sanity/rstcheck/ignore-substitutions.txt'
+        ignore_file = os.path.join(INSTALL_ROOT, 'test/sanity/rstcheck/ignore-substitutions.txt')
         ignore_substitutions = sorted(set(read_lines_without_comments(ignore_file, remove_blank_lines=True)))
 
         paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] in ('.rst',))
diff --git a/test/runner/lib/sanity/shellcheck.py b/test/runner/lib/sanity/shellcheck.py
index 229391e304..0e9f7834d6 100644
--- a/test/runner/lib/sanity/shellcheck.py
+++ b/test/runner/lib/sanity/shellcheck.py
@@ -36,10 +36,10 @@ class ShellcheckTest(SanitySingleVersion):
         :rtype: TestResult
         """
         skip_file = 'test/sanity/shellcheck/skip.txt'
-        skip_paths = set(read_lines_without_comments(skip_file, remove_blank_lines=True))
+        skip_paths = set(read_lines_without_comments(skip_file, remove_blank_lines=True, optional=True))
 
         exclude_file = 'test/sanity/shellcheck/exclude.txt'
-        exclude = set(read_lines_without_comments(exclude_file, remove_blank_lines=True))
+        exclude = set(read_lines_without_comments(exclude_file, remove_blank_lines=True, optional=True))
 
         paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] == '.sh' and i.path not in skip_paths)
 
diff --git a/test/runner/lib/sanity/yamllint.py b/test/runner/lib/sanity/yamllint.py
index d82175dd88..ff495c26df 100644
--- a/test/runner/lib/sanity/yamllint.py
+++ b/test/runner/lib/sanity/yamllint.py
@@ -16,6 +16,7 @@ from lib.util import (
     SubprocessError,
     run_command,
     display,
+    INSTALL_ROOT,
 )
 
 from lib.config import (
@@ -71,7 +72,7 @@ class YamllintTest(SanitySingleVersion):
         """
         cmd = [
             args.python_executable,
-            'test/sanity/yamllint/yamllinter.py',
+            os.path.join(INSTALL_ROOT, 'test/sanity/yamllint/yamllinter.py'),
         ]
 
         data = '\n'.join(paths)
diff --git a/test/runner/lib/types.py b/test/runner/lib/types.py
new file mode 100644
index 0000000000..c1c278c229
--- /dev/null
+++ b/test/runner/lib/types.py
@@ -0,0 +1,19 @@
+"""Import wrapper for type hints when available."""
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+try:
+    from typing import (
+        Any,
+        Dict,
+        FrozenSet,
+        Iterable,
+        List,
+        Optional,
+        Set,
+        Tuple,
+        Type,
+        TypeVar,
+    )
+except ImportError:
+    pass
diff --git a/test/runner/lib/util.py b/test/runner/lib/util.py
index daa92c2138..4dba50aef2 100644
--- a/test/runner/lib/util.py
+++ b/test/runner/lib/util.py
@@ -7,7 +7,6 @@ import contextlib
 import errno
 import fcntl
 import inspect
-import json
 import os
 import pkgutil
 import random
@@ -43,6 +42,14 @@ try:
 except ImportError:
     from pipes import quote as cmd_quote
 
+import lib.types as t
+
+try:
+    C = t.TypeVar('C')
+except AttributeError:
+    C = None
+
+
 DOCKER_COMPLETION = {}  # type: dict[str, dict[str, str]]
 REMOTE_COMPLETION = {}  # type: dict[str, dict[str, str]]
 PYTHON_PATHS = {}  # type: dict[str, str]
@@ -55,6 +62,8 @@ except AttributeError:
 COVERAGE_CONFIG_PATH = '.coveragerc'
 COVERAGE_OUTPUT_PATH = 'coverage'
 
+INSTALL_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
 # Modes are set to allow all users the same level of access.
 # This permits files to be used in tests that change users.
 # The only exception is write access to directories for the user creating them.
@@ -91,7 +100,7 @@ def get_parameterized_completion(cache, name):
     :rtype: dict[str, dict[str, str]]
     """
     if not cache:
-        images = read_lines_without_comments('test/runner/completion/%s.txt' % name, remove_blank_lines=True)
+        images = read_lines_without_comments(os.path.join(INSTALL_ROOT, 'test/runner/completion/%s.txt' % name), remove_blank_lines=True)
 
         cache.update(dict(kvp for kvp in [parse_parameterized_completion(i) for i in images] if kvp))
 
@@ -129,12 +138,15 @@ def remove_file(path):
         os.remove(path)
 
 
-def read_lines_without_comments(path, remove_blank_lines=False):
+def read_lines_without_comments(path, remove_blank_lines=False, optional=False):  # type: (str, bool, bool) -> t.List[str]
     """
-    :type path: str
-    :type remove_blank_lines: bool
-    :rtype: list[str]
+    Returns lines from the specified text file with comments removed.
+    Comments are any content from a hash symbol to the end of a line.
+    Any spaces immediately before a comment are also removed.
     """
+    if optional and not os.path.exists(path):
+        return []
+
     with open(path, 'r') as path_fd:
         lines = path_fd.read().splitlines()
 
@@ -236,7 +248,7 @@ def get_coverage_environment(args, target_name, version, temp_path, module_cover
     else:
         # unit tests, sanity tests and other special cases (localhost only)
         # config and results are in the source tree
-        coverage_config_base_path = os.getcwd()
+        coverage_config_base_path = args.coverage_config_base_path or INSTALL_ROOT
         coverage_output_base_path = os.path.abspath(os.path.join('test/results'))
 
     config_file = os.path.join(coverage_config_base_path, COVERAGE_CONFIG_PATH)
@@ -365,7 +377,7 @@ def intercept_command(args, cmd, target_name, env, capture=False, data=None, cwd
     cmd = list(cmd)
     version = python_version or args.python_version
     interpreter = virtualenv or find_python(version)
-    inject_path = os.path.abspath('test/runner/injector')
+    inject_path = os.path.join(INSTALL_ROOT, 'test/runner/injector')
 
     if not virtualenv:
         # injection of python into the path is required when not activating a virtualenv
@@ -937,11 +949,8 @@ def get_available_port():
         return socket_fd.getsockname()[1]
 
 
-def get_subclasses(class_type):
-    """
-    :type class_type: type
-    :rtype: set[str]
-    """
+def get_subclasses(class_type):  # type: (t.Type[C]) -> t.Set[t.Type[C]]
+    """Returns the set of types that are concrete subclasses of the given type."""
     subclasses = set()
     queue = [class_type]
 
@@ -957,26 +966,59 @@ def get_subclasses(class_type):
     return subclasses
 
 
-def import_plugins(directory):
+def is_subdir(candidate_path, path):  # type: (str, str) -> bool
+    """Returns true if candidate_path is path or a subdirectory of path."""
+    if not path.endswith(os.sep):
+        path += os.sep
+
+    if not candidate_path.endswith(os.sep):
+        candidate_path += os.sep
+
+    return candidate_path.startswith(path)
+
+
+def import_plugins(directory, root=None):  # type: (str, t.Optional[str]) -> None
     """
-    :type directory: str
+    Import plugins from the given directory relative to the given root.
+    If the root is not provided, the 'lib' directory for the test runner will be used.
     """
-    path = os.path.join(os.path.dirname(__file__), directory)
-    prefix = 'lib.%s.' % directory
+    if root is None:
+        root = os.path.dirname(__file__)
+
+    path = os.path.join(root, directory)
+    prefix = 'lib.%s.' % directory.replace(os.sep, '.')
 
     for (_, name, _) in pkgutil.iter_modules([path], prefix=prefix):
-        __import__(name)
+        module_path = os.path.join(root, name[4:].replace('.', os.sep) + '.py')
+        load_module(module_path, name)
 
 
-def load_plugins(base_type, database):
+def load_plugins(base_type, database):  # type: (t.Type[C], t.Dict[str, t.Type[C]]) -> None
     """
-    :type base_type: type
-    :type database: dict[str, type]
+    Load plugins of the specified type and track them in the specified database.
+    Only plugins which have already been imported will be loaded.
     """
-    plugins = dict((sc.__module__.split('.')[2], sc) for sc in get_subclasses(base_type))  # type: dict [str, type]
+    plugins = dict((sc.__module__.split('.')[2], sc) for sc in get_subclasses(base_type))  # type: t.Dict[str, t.Type[C]]
 
     for plugin in plugins:
         database[plugin] = plugins[plugin]
 
 
+def load_module(path, name):  # type: (str, str) -> None
+    """Load a Python module using the given name and path."""
+    if sys.version_info >= (3, 4):
+        import importlib.util
+
+        spec = importlib.util.spec_from_file_location(name, path)
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+
+        sys.modules[name] = module
+    else:
+        import imp
+
+        with open(path, 'r') as module_file:
+            imp.load_module(name, module_file, path, ('.py', 'r', imp.PY_SOURCE))
+
+
 display = Display()  # pylint: disable=locally-disabled, invalid-name
diff --git a/test/sanity/code-smell/shebang.py b/test/sanity/code-smell/shebang.py
index 0d50ff0fdc..04cfed8a39 100755
--- a/test/sanity/code-smell/shebang.py
+++ b/test/sanity/code-smell/shebang.py
@@ -74,6 +74,8 @@ def main():
             is_module = False
             is_integration = False
 
+            dirname = os.path.dirname(path)
+
             if path.startswith('lib/ansible/modules/'):
                 is_module = True
             elif path.startswith('lib/') or path.startswith('test/runner/lib/'):
@@ -87,14 +89,14 @@ def main():
             elif path.startswith('test/integration/targets/'):
                 is_integration = True
 
-                dirname = os.path.dirname(path)
-
                 if dirname.endswith('/library') or dirname.endswith('/plugins/modules') or dirname in (
                     # non-standard module library directories
                     'test/integration/targets/module_precedence/lib_no_extension',
                     'test/integration/targets/module_precedence/lib_with_extension',
                 ):
                     is_module = True
+            elif dirname == 'plugins/modules':
+                is_module = True
 
             if is_module:
                 if executable:
diff --git a/test/sanity/pylint/config/ansible-test b/test/sanity/pylint/config/ansible-test
index 14be00b2c7..ad70665876 100644
--- a/test/sanity/pylint/config/ansible-test
+++ b/test/sanity/pylint/config/ansible-test
@@ -31,6 +31,8 @@ good-names=i,
            k,
            ex,
            Run,
+           C,
+           __metaclass__,
 
 method-rgx=[a-z_][a-z0-9_]{2,40}$
 function-rgx=[a-z_][a-z0-9_]{2,40}$
diff --git a/test/sanity/yamllint/yamllinter.py b/test/sanity/yamllint/yamllinter.py
index 2f74450a28..8e61e4fc89 100755
--- a/test/sanity/yamllint/yamllinter.py
+++ b/test/sanity/yamllint/yamllinter.py
@@ -38,9 +38,11 @@ class YamlChecker(object):
         """
         :type paths: str
         """
-        yaml_conf = YamlLintConfig(file='test/sanity/yamllint/config/default.yml')
-        module_conf = YamlLintConfig(file='test/sanity/yamllint/config/modules.yml')
-        plugin_conf = YamlLintConfig(file='test/sanity/yamllint/config/plugins.yml')
+        config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'config')
+
+        yaml_conf = YamlLintConfig(file=os.path.join(config_path, 'default.yml'))
+        module_conf = YamlLintConfig(file=os.path.join(config_path, 'modules.yml'))
+        plugin_conf = YamlLintConfig(file=os.path.join(config_path, 'plugins.yml'))
 
         for path in paths:
             extension = os.path.splitext(path)[1]