install a randomized RSA key for controller -> isolated rampart auth

see: #6507

6 files changed, 110 insertions, 31 deletions

diff --git a/Makefile b/Makefile
index 4412ba32d1..1207c67cfe 100644
--- a/Makefile
+++ b/Makefile
@@ -353,6 +353,7 @@ init:
 	if [ "$(EXTRA_GROUP_QUEUES)" == "thepentagon" ]; then \
 		tower-manage register_instance --hostname=isolated; \
 		tower-manage register_queue --queuename='thepentagon' --hostnames=isolated --controller=tower; \
+		tower-manage generate_isolated_key | ssh -o "StrictHostKeyChecking no" root@isolated 'cat > /root/.ssh/authorized_keys'; \
 	elif [ "$(EXTRA_GROUP_QUEUES)" != "" ]; then \
 		tower-manage register_queue --queuename=$(EXTRA_GROUP_QUEUES) --hostnames=$(COMPOSE_HOST); \
 	fi;
diff --git a/awx/main/conf.py b/awx/main/conf.py
index dc888ca43b..0015e768d0 100644
--- a/awx/main/conf.py
+++ b/awx/main/conf.py
@@ -174,6 +174,29 @@ register(
 )
 
 register(
+    'AWX_ISOLATED_PRIVATE_KEY',
+    field_class=fields.CharField,
+    default='',
+    allow_blank=True,
+    encrypted=True,
+    label=_('The RSA private key for SSH traffic to isolated instances'),
+    help_text=_('The RSA private key for SSH traffic to isolated instances'),  # noqa
+    category=_('Jobs'),
+    category_slug='jobs',
+)
+
+register(
+    'AWX_ISOLATED_PUBLIC_KEY',
+    field_class=fields.CharField,
+    default='',
+    allow_blank=True,
+    label=_('The RSA public key for SSH traffic to isolated instances'),
+    help_text=_('The RSA public key for SSH traffic to isolated instances'),  # noqa
+    category=_('Jobs'),
+    category_slug='jobs',
+)
+
+register(
     'STDOUT_MAX_BYTES_DISPLAY',
     field_class=fields.IntegerField,
     min_value=0,
diff --git a/awx/main/isolated/isolated_manager.py b/awx/main/isolated/isolated_manager.py
index 139231271b..59ea1229e7 100644
--- a/awx/main/isolated/isolated_manager.py
+++ b/awx/main/isolated/isolated_manager.py
@@ -5,7 +5,9 @@ import StringIO
 import json
 import os
 import re
+import shutil
 import stat
+import tempfile
 import time
 import logging
 
@@ -141,7 +143,7 @@ class IsolatedManager(object):
             args.append('-%s' % ('v' * min(5, self.instance.verbosity)))
         buff = StringIO.StringIO()
         logger.debug('Starting job on isolated host with `run_isolated.yml` playbook.')
-        status, rc = run.run_pexpect(
+        status, rc = IsolatedManager.run_pexpect(
             args, self.awx_playbook_path(), self.env, buff,
             expect_passwords={
                 re.compile(r'Secret:\s*?$', re.M): base64.b64encode(json.dumps(secrets))
@@ -154,6 +156,22 @@ class IsolatedManager(object):
             self.stdout_handle.write(buff.getvalue())
         return status, rc
 
+    @classmethod
+    def run_pexpect(cls, pexpect_args, *args, **kw):
+        isolated_ssh_path = None
+        try:
+            if getattr(settings, 'AWX_ISOLATED_PRIVATE_KEY', None):
+                isolated_ssh_path = tempfile.mkdtemp(prefix='ansible_tower_isolated')
+                os.chmod(isolated_ssh_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
+                isolated_key = os.path.join(isolated_ssh_path, '.isolated')
+                ssh_sock = os.path.join(isolated_ssh_path, '.isolated_ssh_auth.sock')
+                run.open_fifo_write(isolated_key, settings.AWX_ISOLATED_PRIVATE_KEY)
+                pexpect_args = run.wrap_args_with_ssh_agent(pexpect_args, isolated_key, ssh_sock)
+            return run.run_pexpect(pexpect_args, *args, **kw)
+        finally:
+            if isolated_ssh_path:
+                shutil.rmtree(isolated_ssh_path)
+
     def build_isolated_job_data(self):
         '''
         Write the playbook and metadata into a collection of files on the local
@@ -251,7 +269,7 @@ class IsolatedManager(object):
 
             buff = cStringIO.StringIO()
             logger.debug('Checking job on isolated host with `check_isolated.yml` playbook.')
-            status, rc = run.run_pexpect(
+            status, rc = IsolatedManager.run_pexpect(
                 args, self.awx_playbook_path(), self.env, buff,
                 cancelled_callback=self.cancelled_callback,
                 idle_timeout=remaining,
@@ -302,7 +320,7 @@ class IsolatedManager(object):
                 json.dumps(extra_vars)]
         logger.debug('Cleaning up job on isolated host with `clean_isolated.yml` playbook.')
         buff = cStringIO.StringIO()
-        status, rc = run.run_pexpect(
+        status, rc = IsolatedManager.run_pexpect(
             args, self.awx_playbook_path(), self.env, buff,
             idle_timeout=60, job_timeout=60,
             pexpect_timeout=5
@@ -333,7 +351,7 @@ class IsolatedManager(object):
         env['ANSIBLE_STDOUT_CALLBACK'] = 'json'
 
         buff = cStringIO.StringIO()
-        status, rc = run.run_pexpect(
+        status, rc = IsolatedManager.run_pexpect(
             args, cls.awx_playbook_path(), env, buff,
             idle_timeout=60, job_timeout=60,
             pexpect_timeout=5
diff --git a/awx/main/management/commands/generate_isolated_key.py b/awx/main/management/commands/generate_isolated_key.py
new file mode 100644
index 0000000000..225fddc44a
--- /dev/null
+++ b/awx/main/management/commands/generate_isolated_key.py
@@ -0,0 +1,45 @@
+# Copyright (c) 2015 Ansible, Inc.
+# All Rights Reserved
+import datetime
+import sys
+
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import rsa
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from awx.conf.models import Setting
+
+
+class Command(BaseCommand):
+    """Generate and store a randomized RSA key for SSH traffic to isolated instances"""
+    help = 'Generates and stores a randomized RSA key for SSH traffic to isolated instances'
+
+    def handle(self, *args, **kwargs):
+        if getattr(settings, 'AWX_ISOLATED_PRIVATE_KEY', False):
+            print settings.AWX_ISOLATED_PUBLIC_KEY
+            sys.exit(1)
+
+        key = rsa.generate_private_key(
+            public_exponent=65537,
+            key_size=4096,
+            backend=default_backend()
+        )
+        Setting.objects.create(
+            key='AWX_ISOLATED_PRIVATE_KEY',
+            value=key.private_bytes(
+                encoding=serialization.Encoding.PEM,
+                format=serialization.PrivateFormat.TraditionalOpenSSL,
+                encryption_algorithm=serialization.NoEncryption()
+            )
+        ).save()
+        pemfile = Setting.objects.create(
+            key='AWX_ISOLATED_PUBLIC_KEY',
+            value=key.public_key().public_bytes(
+                encoding=serialization.Encoding.OpenSSH,
+                format=serialization.PublicFormat.OpenSSH
+            ) + " generated-by-awx@%s" % datetime.datetime.utcnow().isoformat()
+        )
+        pemfile.save()
+        print pemfile.value
diff --git a/docs/clustering.md b/docs/clustering.md
index 0c0d2b0ab6..1ab85f0ac7 100644
--- a/docs/clustering.md
+++ b/docs/clustering.md
@@ -140,12 +140,11 @@ controller=security
 ```
 
 In the isolated rampart model, "controller" instances interact with "isolated"
-instances via a series of Ansible playbooks over SSH.  As such, all isolated instances
-must be preconfigured by the installer with passwordless SSH access from any potential
-controller instances.  In the example above, the `isolatedA` and `isolatedB` hosts
-must be reachable from `towerB` and `towerC` hosts via `ssh
-awx@<isolated-hostname>` (meaning, `authorized_keys` must be pre-distributed to
-the `isolatedA` and `isolatedB` hosts).
+instances via a series of Ansible playbooks over SSH.  At installation time,
+a randomized RSA key is generated and distributed as an authorized key to all
+"isolated" instances.  The private half of the key is encrypted and stored
+within Tower, and is used to authenticate from "controller" instances to
+"isolated" instances when jobs are run.
 
 When a job is scheduled to run on an "isolated" instance:
 
@@ -185,6 +184,20 @@ Recommendations for system configuration with isolated groups:
    variable - the behavior in this case can not be predicted.
  - Do not put an isolated instance in more than 1 isolated group.
 
+Isolated Node Authentication
+----------------------------
+By default - at installation time - a randomized RSA key is generated and
+distributed as an authorized key to all "isolated" instances.  The private half
+of the key is encrypted and stored within Tower, and is used to authenticate
+from "controller" instances to "isolated" instances when jobs are run.
+
+For users who wish to manage SSH authentication from controlling nodes to
+isolated nodes via some system _outside_ of Tower (such as externally-managed
+passwordless SSH keys), this behavior can be disabled by unsetting two Tower
+API settings values:
+
+`HTTP PATCH /api/v2/settings/jobs/ {'AWX_ISOLATED_PRIVATE_KEY': '', 'AWX_ISOLATED_PUBLIC_KEY': ''}`
+
 
 ### Provisioning and Deprovisioning Instances and Groups
 
diff --git a/tools/docker-isolated/README.md b/tools/docker-isolated/README.md
index 897103b18b..9a2ae470c2 100644
--- a/tools/docker-isolated/README.md
+++ b/tools/docker-isolated/README.md
@@ -40,27 +40,6 @@ and they are structured as follows:
 The `controller` for the group "thepentagon" and all hosts therein is
 determined by a ForeignKey within the instance group.
 
-## Development Testing Notes
-
-### Test the SSH connection between containers
-
-While the environment is running, you can test the connection like so:
-
-```bash
-docker exec -i -t tools_tower_1 /bin/bash
-```
-
-Inside the context of that container:
-
-```bash
-ssh root@isolated
-```
-
-(note: awx user has been deprecated)
-
-This should give a shell to the `tools_isolated_1` container, as the
-`tools_tower_1` container sees it.
-
 ### Run a playbook
 
 In order to run an isolated job, associate the instance group `thepentagon` with