Install ansible-test (#60718)

* Install ansible-test

Modify the install script to install ansible-test and its supporting
code.  Alternative to #60701 that doesn't change package_dir ansible for
fear that it might regress https://github.com/ansible/ansible/issues/10437

Also:
* No longer use package_data.  Everything in the package dirs is going
  to be installed.  Anything that shouldn't be installed needs to be
  moved elsewhere.
* modify the algorithm to store symlinks which are in the same tree
  instead of same directory

* Add ansible_test files to package-data sanity test

* MANIFEST.in cleanups

* Add lib/ansible/config/*.yml
* Make most things in code directories (lib/ansible and test/lib/ansible_test/)
  use explicit file extensions instead of wildcards for maintainability
* Exclude common file extensions that we don't want included in the code
  directories

* Change package-data test to be more complete

* Now compares the repository, sdist, and install
* Compares both that everything in the sdist is in the repo and
  everything in the install is in the sdist in addition to comparing that
  everything in the repo that we want is in the install

* Leave out test artifacts

Only include the directory structure for test/results and test/cache not
any files that may have been generated by test runs

Remove test/utils files from the sdist as these are only needed for our CI

cleanup of docs in MANIFEST.in; getting rid of build files.

* Add the ability to output sdist and snapshot to specific directory

* Add a warning about modifying the heuristic to setup.py

* Address generated files

* Use make snapshot instead of sdist to generate changelog and man pages
  and make sure they're included
* Ignore both the test/utils and generated test files (results, cache)
* Deal with Python3 __pycache__ byte code caches
* Don't check documentation, that isn't built for the sdist
* Restructure for clarity

* Add cli web docs to make clean

This was causing problems when attempting to test that the sdist didn't
have extra files

* Fix bug constructing python names from __pycache__ names

* Create a clean repo to work from

* Exclude test/legacy and be more explicit on extensions

* Exclude the legacy directory from sdist

1 files changed, 45 insertions, 24 deletions

diff --git a/setup.py b/setup.py
index c607de5fa8..e302bcdae9 100644
--- a/setup.py
+++ b/setup.py
@@ -36,6 +36,13 @@ def _find_symlinks(topdir, extension=''):
     Maintained symlinks exist in the bin dir or are modules which have
     aliases.  Our heuristic is that they are a link in a certain path which
     point to a file in the same directory.
+
+    .. warn::
+
+        We want the symlinks in :file:`bin/` that link into :file:`lib/ansible/*` (currently,
+        :command:`ansible`, :command:`ansible-test`, and :command:`ansible-connection`) to become
+        real files on install.  Updates to the heuristic here *must not* add them to the symlink
+        cache.
     """
     symlinks = defaultdict(list)
     for base_path, dirs, files in os.walk(topdir):
@@ -43,11 +50,36 @@ def _find_symlinks(topdir, extension=''):
             filepath = os.path.join(base_path, filename)
             if os.path.islink(filepath) and filename.endswith(extension):
                 target = os.readlink(filepath)
+                if target.startswith('/'):
+                    # We do not support absolute symlinks at all
+                    continue
+
                 if os.path.dirname(target) == '':
                     link = filepath[len(topdir):]
                     if link.startswith('/'):
                         link = link[1:]
                     symlinks[os.path.basename(target)].append(link)
+                else:
+                    # Count how many directory levels from the topdir we are
+                    levels_deep = os.path.dirname(filepath).count('/')
+
+                    # Count the number of directory levels higher we walk up the tree in target
+                    target_depth = 0
+                    for path_component in target.split('/'):
+                        if path_component == '..':
+                            target_depth += 1
+                            # If we walk past the topdir, then don't store
+                            if target_depth >= levels_deep:
+                                break
+                        else:
+                            target_depth -= 1
+                    else:
+                        # If we managed to stay within the tree, store the symlink
+                        link = filepath[len(topdir):]
+                        if link.startswith('/'):
+                            link = link[1:]
+                        symlinks[target].append(link)
+
     return symlinks
 
 
@@ -69,8 +101,11 @@ def _maintain_symlinks(symlink_type, base_path):
             # SYMLINKS_CACHE doesn't exist.  Fallback to trying to create the
             # cache now.  Will work if we're running directly from a git
             # checkout or from an sdist created earlier.
+            library_symlinks = _find_symlinks('lib', '.py')
+            library_symlinks.update(_find_symlinks('test/lib'))
+
             symlink_data = {'script': _find_symlinks('bin'),
-                            'library': _find_symlinks('lib', '.py'),
+                            'library': library_symlinks,
                             }
 
             # Sanity check that something we know should be a symlink was
@@ -129,8 +164,11 @@ class SDistCommand(SDist):
     def run(self):
         # have to generate the cache of symlinks for release as sdist is the
         # only command that has access to symlinks from the git repo
+        library_symlinks = _find_symlinks('lib', '.py')
+        library_symlinks.update(_find_symlinks('test/lib'))
+
         symlinks = {'script': _find_symlinks('bin'),
-                    'library': _find_symlinks('lib', '.py'),
+                    'library': library_symlinks,
                     }
         _cache_symlinks(symlinks)
 
@@ -254,28 +292,10 @@ static_setup_params = dict(
     # Ansible will also make use of a system copy of python-six and
     # python-selectors2 if installed but use a Bundled copy if it's not.
     python_requires='>=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*',
-    package_dir={'': 'lib'},
-    packages=find_packages('lib'),
-    package_data={
-        '': [
-            'executor/powershell/*.ps1',
-            'module_utils/csharp/*.cs',
-            'module_utils/csharp/*/*.cs',
-            'module_utils/powershell/*.psm1',
-            'module_utils/powershell/*/*.psm1',
-            'modules/windows/*.ps1',
-            'modules/windows/*/*.ps1',
-            'galaxy/data/*.*',
-            'galaxy/data/*/*.*',
-            'galaxy/data/*/.*',
-            'galaxy/data/*/*/.*',
-            'galaxy/data/*/*/*.*',
-            'galaxy/data/*/tests/inventory',
-            'galaxy/data/*/role/tests/inventory',
-            'config/base.yml',
-            'config/module_defaults.yml',
-        ],
-    },
+    package_dir={'': 'lib',
+                 'ansible_test': 'test/lib/ansible_test'},
+    packages=find_packages('lib') + find_packages('test/lib'),
+    include_package_data=True,
     classifiers=[
         'Development Status :: 5 - Production/Stable',
         'Environment :: Console',
@@ -306,6 +326,7 @@ static_setup_params = dict(
         'bin/ansible-vault',
         'bin/ansible-config',
         'bin/ansible-inventory',
+        'bin/ansible-test',
     ],
     data_files=[],
     # Installing as zip files would break due to references to __file__