diff options
22 files changed, 493 insertions, 204 deletions
diff --git a/awx/conf/migrations/0004_v320_reencrypt.py b/awx/conf/migrations/0004_v320_reencrypt.py new file mode 100644 index 0000000000..4a68ccd088 --- /dev/null +++ b/awx/conf/migrations/0004_v320_reencrypt.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations +from awx.conf.migrations import _reencrypt + + +class Migration(migrations.Migration): + + dependencies = [ + ('conf', '0003_v310_JSONField_changes'), + ] + + operations = [ + migrations.RunPython(_reencrypt.replace_aesecb_fernet), + ] diff --git a/awx/conf/migrations/_reencrypt.py b/awx/conf/migrations/_reencrypt.py new file mode 100644 index 0000000000..013c96d33b --- /dev/null +++ b/awx/conf/migrations/_reencrypt.py @@ -0,0 +1,102 @@ +import base64 +import hashlib + +import six +from django.utils.encoding import smart_str +from Crypto.Cipher import AES + +from awx.conf import settings_registry + + +__all__ = ['replace_aesecb_fernet', 'get_encryption_key', 'encrypt_field', + 'decrypt_value', 'decrypt_value'] + + +def replace_aesecb_fernet(apps, schema_editor): + Setting = apps.get_model('conf', 'Setting') + + for setting in Setting.objects.filter().order_by('pk'): + if settings_registry.is_setting_encrypted(setting.key): + if setting.value.startswith('$encrypted$AESCBC$'): + continue + setting.value = decrypt_field(setting, 'value') + setting.save() + + +def get_encryption_key(field_name, pk=None): + ''' + Generate key for encrypted password based on field name, + ``settings.SECRET_KEY``, and instance pk (if available). + + :param pk: (optional) the primary key of the ``awx.conf.model.Setting``; + can be omitted in situations where you're encrypting a setting + that is not database-persistent (like a read-only setting) + ''' + from django.conf import settings + h = hashlib.sha1() + h.update(settings.SECRET_KEY) + if pk is not None: + h.update(str(pk)) + h.update(field_name) + return h.digest()[:16] + + +def decrypt_value(encryption_key, value): + raw_data = value[len('$encrypted$'):] + # If the encrypted string contains a UTF8 marker, discard it + utf8 = raw_data.startswith('UTF8$') + if utf8: + raw_data = raw_data[len('UTF8$'):] + algo, b64data = raw_data.split('$', 1) + if algo != 'AES': + raise ValueError('unsupported algorithm: %s' % algo) + encrypted = base64.b64decode(b64data) + cipher = AES.new(encryption_key, AES.MODE_ECB) + value = cipher.decrypt(encrypted) + value = value.rstrip('\x00') + # If the encrypted string contained a UTF8 marker, decode the data + if utf8: + value = value.decode('utf-8') + return value + + +def decrypt_field(instance, field_name, subfield=None): + ''' + Return content of the given instance and field name decrypted. + ''' + value = getattr(instance, field_name) + if isinstance(value, dict) and subfield is not None: + value = value[subfield] + if not value or not value.startswith('$encrypted$'): + return value + key = get_encryption_key(field_name, getattr(instance, 'pk', None)) + + return decrypt_value(key, value) + + +def encrypt_field(instance, field_name, ask=False, subfield=None, skip_utf8=False): + ''' + Return content of the given instance and field name encrypted. + ''' + value = getattr(instance, field_name) + if isinstance(value, dict) and subfield is not None: + value = value[subfield] + if not value or value.startswith('$encrypted$') or (ask and value == 'ASK'): + return value + if skip_utf8: + utf8 = False + else: + utf8 = type(value) == six.text_type + value = smart_str(value) + key = get_encryption_key(field_name, getattr(instance, 'pk', None)) + cipher = AES.new(key, AES.MODE_ECB) + while len(value) % cipher.block_size != 0: + value += '\x00' + encrypted = cipher.encrypt(value) + b64data = base64.b64encode(encrypted) + tokens = ['$encrypted', 'AES', b64data] + if utf8: + # If the value to encrypt is utf-8, we need to add a marker so we + # know to decode the data when it's decrypted later + tokens.insert(1, 'UTF8') + return '$'.join(tokens) diff --git a/awx/conf/settings.py b/awx/conf/settings.py index 616ed1fcd3..68f63070e3 100644 --- a/awx/conf/settings.py +++ b/awx/conf/settings.py @@ -52,7 +52,7 @@ SETTING_CACHE_TIMEOUT = 60 # Flag indicating whether to store field default values in the cache. SETTING_CACHE_DEFAULTS = True -__all__ = ['SettingsWrapper'] +__all__ = ['SettingsWrapper', 'get_settings_to_cache', 'SETTING_CACHE_NOTSET'] @contextlib.contextmanager @@ -147,6 +147,27 @@ class EncryptedCacheProxy(object): setattr(self.cache, name, value) +def get_writeable_settings(registry): + return registry.get_registered_settings(read_only=False) + + +def get_settings_to_cache(registry): + return dict([(key, SETTING_CACHE_NOTSET) for key in get_writeable_settings(registry)]) + + +def get_cache_value(value): + '''Returns the proper special cache setting for a value + based on instance type. + ''' + if value is None: + value = SETTING_CACHE_NONE + elif isinstance(value, (list, tuple)) and len(value) == 0: + value = SETTING_CACHE_EMPTY_LIST + elif isinstance(value, (dict,)) and len(value) == 0: + value = SETTING_CACHE_EMPTY_DICT + return value + + class SettingsWrapper(UserSettingsHolder): @classmethod @@ -188,18 +209,6 @@ class SettingsWrapper(UserSettingsHolder): def _get_supported_settings(self): return self.registry.get_registered_settings() - def _get_writeable_settings(self): - return self.registry.get_registered_settings(read_only=False) - - def _get_cache_value(self, value): - if value is None: - value = SETTING_CACHE_NONE - elif isinstance(value, (list, tuple)) and len(value) == 0: - value = SETTING_CACHE_EMPTY_LIST - elif isinstance(value, (dict,)) and len(value) == 0: - value = SETTING_CACHE_EMPTY_DICT - return value - def _preload_cache(self): # Ensure we're only modifying local preload timeout from one thread. with self._awx_conf_preload_lock: @@ -212,7 +221,7 @@ class SettingsWrapper(UserSettingsHolder): # make those read-only to avoid overriding in the database. if not self._awx_conf_init_readonly and 'migrate_to_database_settings' not in sys.argv: defaults_snapshot = self._get_default('DEFAULTS_SNAPSHOT') - for key in self._get_writeable_settings(): + for key in get_writeable_settings(self.registry): init_default = defaults_snapshot.get(key, None) try: file_default = self._get_default(key) @@ -230,7 +239,7 @@ class SettingsWrapper(UserSettingsHolder): # Initialize all database-configurable settings with a marker value so # to indicate from the cache that the setting is not configured without # a database lookup. - settings_to_cache = dict([(key, SETTING_CACHE_NOTSET) for key in self._get_writeable_settings()]) + settings_to_cache = get_settings_to_cache(self.registry) # Load all settings defined in the database. for setting in Setting.objects.filter(key__in=settings_to_cache.keys(), user__isnull=True).order_by('pk'): if settings_to_cache[setting.key] != SETTING_CACHE_NOTSET: @@ -239,7 +248,7 @@ class SettingsWrapper(UserSettingsHolder): value = decrypt_field(setting, 'value') else: value = setting.value - settings_to_cache[setting.key] = self._get_cache_value(value) + settings_to_cache[setting.key] = get_cache_value(value) # Load field default value for any settings not found in the database. if SETTING_CACHE_DEFAULTS: for key, value in settings_to_cache.items(): @@ -247,7 +256,7 @@ class SettingsWrapper(UserSettingsHolder): continue field = self.registry.get_setting_field(key) try: - settings_to_cache[key] = self._get_cache_value(field.get_default()) + settings_to_cache[key] = get_cache_value(field.get_default()) except SkipField: pass # Generate a cache key for each setting and store them all at once. @@ -296,9 +305,9 @@ class SettingsWrapper(UserSettingsHolder): value = SETTING_CACHE_NOTSET if cache_value != value: logger.debug('cache set(%r, %r, %r)', cache_key, - self._get_cache_value(value), + get_cache_value(value), SETTING_CACHE_TIMEOUT) - self.cache.set(cache_key, self._get_cache_value(value), timeout=SETTING_CACHE_TIMEOUT) + self.cache.set(cache_key, get_cache_value(value), timeout=SETTING_CACHE_TIMEOUT) if value == SETTING_CACHE_NOTSET and not SETTING_CACHE_DEFAULTS: try: value = field.get_default() diff --git a/awx/conf/tests/__init__.py b/awx/conf/tests/__init__.py deleted file mode 100644 index 46176c348f..0000000000 --- a/awx/conf/tests/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (c) 2017 Ansible, Inc. -# All Rights Reserved. diff --git a/awx/conf/tests/functional/test_reencrypt_migration.py b/awx/conf/tests/functional/test_reencrypt_migration.py new file mode 100644 index 0000000000..78cbf734a9 --- /dev/null +++ b/awx/conf/tests/functional/test_reencrypt_migration.py @@ -0,0 +1,30 @@ +import pytest +import mock + +from django.apps import apps +from awx.conf.migrations._reencrypt import ( + replace_aesecb_fernet, + encrypt_field, + decrypt_field, +) +from awx.conf.settings import Setting +from awx.main.utils import decrypt_field as new_decrypt_field + + +@pytest.mark.django_db +def test_settings(): + with mock.patch('awx.conf.models.encrypt_field', encrypt_field): + with mock.patch('awx.conf.settings.decrypt_field', decrypt_field): + setting = Setting.objects.create(key='SOCIAL_AUTH_GITHUB_SECRET', value='test') + assert setting.value.startswith('$encrypted$AES$') + + replace_aesecb_fernet(apps, None) + setting.refresh_from_db() + + assert setting.value.startswith('$encrypted$AESCBC$') + assert new_decrypt_field(setting, 'value') == 'test' + + # This is here for a side-effect. + # Exception if the encryption type of AESCBC is not properly skipped, ensures + # our `startswith` calls don't have typos + replace_aesecb_fernet(apps, None) diff --git a/awx/main/fields.py b/awx/main/fields.py index ab032ced84..325ae25967 100644 --- a/awx/main/fields.py +++ b/awx/main/fields.py @@ -472,7 +472,7 @@ class CredentialInputField(JSONSchemaField): v != '$encrypted$', model_instance.pk ]): - decrypted_values[k] = utils.common.decrypt_field(model_instance, k) + decrypted_values[k] = utils.decrypt_field(model_instance, k) else: decrypted_values[k] = v diff --git a/awx/main/migrations/0038_v320_release.py b/awx/main/migrations/0038_v320_release.py index 38df95682f..04c9a08806 100644 --- a/awx/main/migrations/0038_v320_release.py +++ b/awx/main/migrations/0038_v320_release.py @@ -9,6 +9,7 @@ from psycopg2.extensions import AsIs from django.db import migrations, models # AWX +from awx.main.migrations import _reencrypt as reencrypt import awx.main.fields from awx.main.models import Host @@ -260,7 +261,7 @@ class Migration(migrations.Migration): name='Permission', ), - # Insights + # Insights migrations.AddField( model_name='host', name='insights_system_id', @@ -276,4 +277,5 @@ class Migration(migrations.Migration): name='kind', field=models.CharField(default=b'', help_text='Kind of inventory being represented.', max_length=32, blank=True, choices=[(b'', 'Hosts have a direct link to this inventory.'), (b'smart', 'Hosts for inventory generated using the host_filter property.')]), ), + migrations.RunPython(reencrypt.replace_aesecb_fernet), ] diff --git a/awx/main/migrations/0044_v320_reencrypt.py b/awx/main/migrations/0044_v320_reencrypt.py new file mode 100644 index 0000000000..dc72811b71 --- /dev/null +++ b/awx/main/migrations/0044_v320_reencrypt.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations +from awx.main.migrations import _reencrypt + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0043_v320_instancegroups'), + ] + + operations = [ + migrations.RunPython(_reencrypt.replace_aesecb_fernet), + ] diff --git a/awx/main/migrations/_credentialtypes.py b/awx/main/migrations/_credentialtypes.py index 4dd083fa0b..a0fc1aac1e 100644 --- a/awx/main/migrations/_credentialtypes.py +++ b/awx/main/migrations/_credentialtypes.py @@ -1,6 +1,6 @@ from awx.main import utils from awx.main.models import CredentialType -from awx.main.utils.common import encrypt_field, decrypt_field +from awx.main.utils import encrypt_field, decrypt_field from django.db.models import Q diff --git a/awx/main/migrations/_reencrypt.py b/awx/main/migrations/_reencrypt.py new file mode 100644 index 0000000000..29ae5867d1 --- /dev/null +++ b/awx/main/migrations/_reencrypt.py @@ -0,0 +1,46 @@ +from awx.conf.migrations._reencrypt import decrypt_field + + +__all__ = ['replace_aesecb_fernet'] + + +def replace_aesecb_fernet(apps, schema_editor): + _notification_templates(apps) + _credentials(apps) + _unified_jobs(apps) + + +def _notification_templates(apps): + NotificationTemplate = apps.get_model('main', 'NotificationTemplate') + for nt in NotificationTemplate.objects.all(): + for field in filter(lambda x: nt.notification_class.init_parameters[x]['type'] == "password", + nt.notification_class.init_parameters): + if nt.notification_configuration[field].startswith('$encrypted$AESCBC$'): + continue + value = decrypt_field(nt, 'notification_configuration', subfield=field) + nt.notification_configuration[field] = value + nt.save() + + +def _credentials(apps): + Credential = apps.get_model('main', 'Credential') + for credential in Credential.objects.all(): + for field_name, value in credential.inputs.items(): + if field_name in credential.credential_type.secret_fields: + value = getattr(credential, field_name) + if value.startswith('$encrypted$AESCBC$'): + continue + value = decrypt_field(credential, field_name) + credential.inputs[field_name] = value + credential.save() + + +def _unified_jobs(apps): + UnifiedJob = apps.get_model('main', 'UnifiedJob') + for uj in UnifiedJob.objects.all(): + if uj.start_args is not None: + if uj.start_args.startswith('$encrypted$AESCBC$'): + continue + start_args = decrypt_field(uj, 'start_args') + uj.start_args = start_args + uj.save() diff --git a/awx/main/tests/functional/api/test_credential.py b/awx/main/tests/functional/api/test_credential.py index be7521c4fd..e1b5cbe8f9 100644 --- a/awx/main/tests/functional/api/test_credential.py +++ b/awx/main/tests/functional/api/test_credential.py @@ -2,7 +2,7 @@ import mock # noqa import pytest from awx.main.models.credential import Credential, CredentialType -from awx.main.utils.common import decrypt_field +from awx.main.utils import decrypt_field from awx.api.versioning import reverse EXAMPLE_PRIVATE_KEY = '-----BEGIN PRIVATE KEY-----\nxyz==\n-----END PRIVATE KEY-----' diff --git a/awx/main/tests/functional/test_credential.py b/awx/main/tests/functional/test_credential.py index bd2fdbcdae..a288eff1e4 100644 --- a/awx/main/tests/functional/test_credential.py +++ b/awx/main/tests/functional/test_credential.py @@ -4,7 +4,7 @@ import pytest from django.core.exceptions import ValidationError -from awx.main.utils.common import decrypt_field +from awx.main.utils import decrypt_field from awx.main.models import Credential, CredentialType from rest_framework import serializers diff --git a/awx/main/tests/functional/test_credential_migration.py b/awx/main/tests/functional/test_credential_migration.py index 6595e890e8..386839591c 100644 --- a/awx/main/tests/functional/test_credential_migration.py +++ b/awx/main/tests/functional/test_credential_migration.py @@ -7,7 +7,7 @@ from django.apps import apps from awx.main.models import Credential, CredentialType from awx.main.migrations._credentialtypes import migrate_to_v2_credentials -from awx.main.utils.common import decrypt_field +from awx.main.utils import decrypt_field from awx.main.migrations._credentialtypes import _disassociate_non_insights_projects EXAMPLE_PRIVATE_KEY = '-----BEGIN PRIVATE KEY-----\nxyz==\n-----END PRIVATE KEY-----' @@ -319,7 +319,7 @@ def test_insights_migration(): 'username': 'bob', 'password': 'some-password', }) - + assert cred.credential_type.name == 'Insights Basic Auth' assert cred.inputs['username'] == 'bob' assert cred.inputs['password'].startswith('$encrypted$') diff --git a/awx/main/tests/functional/test_reencrypt_migration.py b/awx/main/tests/functional/test_reencrypt_migration.py new file mode 100644 index 0000000000..de594b8638 --- /dev/null +++ b/awx/main/tests/functional/test_reencrypt_migration.py @@ -0,0 +1,82 @@ +import json +import pytest +import mock + +from django.apps import apps + +from awx.main.models import ( + UnifiedJob, + NotificationTemplate, + Credential, +) +from awx.main.models.credential import ssh + +from awx.conf.migrations._reencrypt import encrypt_field +from awx.main.migrations._reencrypt import ( + _notification_templates, + _credentials, + _unified_jobs, +) + +from awx.main.utils import decrypt_field + + +@pytest.mark.django_db +def test_notification_template_migration(): + with mock.patch('awx.main.models.notifications.encrypt_field', encrypt_field): + nt = NotificationTemplate.objects.create(notification_type='slack', notification_configuration=dict(token='test')) + + + assert nt.notification_configuration['token'].startswith('$encrypted$AES$') + + _notification_templates(apps) + nt.refresh_from_db() + + assert nt.notification_configuration['token'].startswith('$encrypted$AESCBC$') + assert decrypt_field(nt, 'notification_configuration', subfield='token') == 'test' + + # This is here for a side-effect. + # Exception if the encryption type of AESCBC is not properly skipped, ensures + # our `startswith` calls don't have typos + _notification_templates(apps) + + +@pytest.mark.django_db +def test_credential_migration(): + with mock.patch('awx.main.models.credential.encrypt_field', encrypt_field): + cred_type = ssh() + cred_type.save() + + cred = Credential.objects.create(credential_type=cred_type, inputs=dict(password='test')) + + assert cred.password.startswith('$encrypted$AES$') + + _credentials(apps) + cred.refresh_from_db() + + assert cred.password.startswith('$encrypted$AESCBC$') + assert decrypt_field(cred, 'password') == 'test' + + # This is here for a side-effect. + # Exception if the encryption type of AESCBC is not properly skipped, ensures + # our `startswith` calls don't have typos + _credentials(apps) + + +@pytest.mark.django_db +def test_unified_job_migration(): + with mock.patch('awx.main.models.base.encrypt_field', encrypt_field): + uj = UnifiedJob.objects.create(launch_type='manual', start_args=json.dumps({'test':'value'})) + + assert uj.start_args.startswith('$encrypted$AES$') + + _unified_jobs(apps) + uj.refresh_from_db() + + assert uj.start_args.startswith('$encrypted$AESCBC$') + assert json.loads(decrypt_field(uj, 'start_args')) == {'test':'value'} + + # This is here for a side-effect. + # Exception if the encryption type of AESCBC is not properly skipped, ensures + # our `startswith` calls don't have typos + _unified_jobs(apps) diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index cb4ce9f909..9b575da56d 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -26,7 +26,7 @@ from awx.main.models import ( from awx.main import tasks from awx.main.task_engine import TaskEnhancer -from awx.main.utils.common import encrypt_field +from awx.main.utils import encrypt_field @contextmanager diff --git a/awx/main/tests/unit/utils/common/test_common.py b/awx/main/tests/unit/utils/common/test_common.py deleted file mode 100644 index 95ba3b8b9f..0000000000 --- a/awx/main/tests/unit/utils/common/test_common.py +++ /dev/null @@ -1,65 +0,0 @@ -# -*- coding: utf-8 -*- - -# Copyright (c) 2017 Ansible, Inc. -# All Rights Reserved. -import pytest - -from awx.conf.models import Setting -from awx.main.utils import common - - -def test_encrypt_field(): - field = Setting(pk=123, value='ANSIBLE') - encrypted = field.value = common.encrypt_field(field, 'value') - assert encrypted == '$encrypted$AES$Ey83gcmMuBBT1OEq2lepnw==' - assert common.decrypt_field(field, 'value') == 'ANSIBLE' - - -def test_encrypt_field_without_pk(): - field = Setting(value='ANSIBLE') - encrypted = field.value = common.encrypt_field(field, 'value') - assert encrypted == '$encrypted$AES$8uIzEoGyY6QJwoTWbMFGhw==' - assert common.decrypt_field(field, 'value') == 'ANSIBLE' - - -def test_encrypt_field_with_unicode_string(): - value = u'Iñtërnâtiônàlizætiøn' - field = Setting(value=value) - encrypted = field.value = common.encrypt_field(field, 'value') - assert encrypted == '$encrypted$UTF8$AES$AESQbqOefpYcLC7x8yZ2aWG4FlXlS66JgavLbDp/DSM=' - assert common.decrypt_field(field, 'value') == value - - -def test_encrypt_field_force_disable_unicode(): - value = u"NothingSpecial" - field = Setting(value=value) - encrypted = field.value = common.encrypt_field(field, 'value', skip_utf8=True) - assert "UTF8" not in encrypted - assert common.decrypt_field(field, 'value') == value - - -def test_encrypt_subfield(): - field = Setting(value={'name': 'ANSIBLE'}) - encrypted = field.value = common.encrypt_field(field, 'value', subfield='name') - assert encrypted == '$encrypted$AES$8uIzEoGyY6QJwoTWbMFGhw==' - assert common.decrypt_field(field, 'value', subfield='name') == 'ANSIBLE' - - -def test_encrypt_field_with_ask(): - encrypted = common.encrypt_field(Setting(value='ASK'), 'value', ask=True) - assert encrypted == 'ASK' - - -def test_encrypt_field_with_empty_value(): - encrypted = common.encrypt_field(Setting(value=None), 'value') - assert encrypted is None - - -@pytest.mark.parametrize('input_, output', [ - ({"foo": "bar"}, {"foo": "bar"}), - ('{"foo": "bar"}', {"foo": "bar"}), - ('---\nfoo: bar', {"foo": "bar"}), - (4399, {}), -]) -def test_parse_yaml_or_json(input_, output): - assert common.parse_yaml_or_json(input_) == output diff --git a/awx/main/tests/unit/utils/test_common.py b/awx/main/tests/unit/utils/test_common.py new file mode 100644 index 0000000000..030962da52 --- /dev/null +++ b/awx/main/tests/unit/utils/test_common.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2017 Ansible, Inc. +# All Rights Reserved. +import pytest + +from awx.main.utils import common + + +@pytest.mark.parametrize('input_, output', [ + ({"foo": "bar"}, {"foo": "bar"}), + ('{"foo": "bar"}', {"foo": "bar"}), + ('---\nfoo: bar', {"foo": "bar"}), + (4399, {}), +]) +def test_parse_yaml_or_json(input_, output): + assert common.parse_yaml_or_json(input_) == output diff --git a/awx/main/tests/unit/utils/test_encryption.py b/awx/main/tests/unit/utils/test_encryption.py new file mode 100644 index 0000000000..29b68cc56b --- /dev/null +++ b/awx/main/tests/unit/utils/test_encryption.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2017 Ansible, Inc. +# All Rights Reserved. +from awx.conf.models import Setting +from awx.main.utils import encryption + + +def test_encrypt_field(): + field = Setting(pk=123, value='ANSIBLE') + encrypted = field.value = encryption.encrypt_field(field, 'value') + assert encryption.decrypt_field(field, 'value') == 'ANSIBLE' + assert encrypted.startswith('$encrypted$AESCBC$') + + +def test_encrypt_field_without_pk(): + field = Setting(value='ANSIBLE') + encrypted = field.value = encryption.encrypt_field(field, 'value') + assert encryption.decrypt_field(field, 'value') == 'ANSIBLE' + assert encrypted.startswith('$encrypted$AESCBC$') + + +def test_encrypt_field_with_unicode_string(): + value = u'Iñtërnâtiônàlizætiøn' + field = Setting(value=value) + encrypted = field.value = encryption.encrypt_field(field, 'value') + assert encryption.decrypt_field(field, 'value') == value + assert encrypted.startswith('$encrypted$UTF8$AESCBC$') + + +def test_encrypt_field_force_disable_unicode(): + value = u"NothingSpecial" + field = Setting(value=value) + encrypted = field.value = encryption.encrypt_field(field, 'value', skip_utf8=True) + assert "UTF8" not in encrypted + assert encryption.decrypt_field(field, 'value') == value + + +def test_encrypt_subfield(): + field = Setting(value={'name': 'ANSIBLE'}) + encrypted = field.value = encryption.encrypt_field(field, 'value', subfield='name') + assert encryption.decrypt_field(field, 'value', subfield='name') == 'ANSIBLE' + assert encrypted.startswith('$encrypted$AESCBC$') + + +def test_encrypt_field_with_ask(): + encrypted = encryption.encrypt_field(Setting(value='ASK'), 'value', ask=True) + assert encrypted == 'ASK' + + +def test_encrypt_field_with_empty_value(): + encrypted = encryption.encrypt_field(Setting(value=None), 'value') + assert encrypted is None diff --git a/awx/main/utils/__init__.py b/awx/main/utils/__init__.py index fbdb9d11e0..fb20b92898 100644 --- a/awx/main/utils/__init__.py +++ b/awx/main/utils/__init__.py @@ -3,22 +3,4 @@ # AWX from awx.main.utils.common import * # noqa - -# Fields that didn't get included in __all__ -# TODO: after initial commit of file move to devel, these can be added -# to common.py __all__ and removed here -from awx.main.utils.common import ( # noqa - RequireDebugTrueOrTest, - encrypt_field, - parse_yaml_or_json, - decrypt_field, - timestamp_apiformat, - model_instance_diff, - model_to_dict, - check_proot_installed, - build_proot_temp_dir, - wrap_args_with_proot, - get_system_task_capacity, - decrypt_field_value, - has_model_field_prefetched -) +from awx.main.utils.encryption import * # noqa diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index dd42078d43..83e27558cb 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -3,7 +3,6 @@ # Python import base64 -import hashlib import json import yaml import logging @@ -21,8 +20,6 @@ import tempfile # Decorator from decorator import decorator -import six - # Django from django.utils.translation import ugettext_lazy as _ from django.db.models.fields.related import ForeignObjectRel, ManyToManyField @@ -33,9 +30,6 @@ from django.utils.encoding import smart_str from django.utils.text import slugify from django.apps import apps -# PyCrypto -from Crypto.Cipher import AES - logger = logging.getLogger('awx.main.utils') __all__ = ['get_object_or_400', 'get_object_or_403', 'camelcase_to_underscore', 'memoize', @@ -45,7 +39,10 @@ __all__ = ['get_object_or_400', 'get_object_or_403', 'camelcase_to_underscore', 'ignore_inventory_computed_fields', 'ignore_inventory_group_removal', '_inventory_updates', 'get_pk_from_dict', 'getattrd', 'NoDefaultProvided', 'get_current_apps', 'set_current_apps', 'OutputEventFilter', - 'callback_filter_out_ansible_extra_vars', 'get_search_fields',] + 'callback_filter_out_ansible_extra_vars', 'get_search_fields', 'get_system_task_capacity', + 'wrap_args_with_proot', 'build_proot_temp_dir', 'check_proot_installed', 'model_to_dict', + 'model_instance_diff', 'timestamp_apiformat', 'parse_yaml_or_json', 'RequireDebugTrueOrTest', + 'has_model_field_prefetched'] def get_object_or_400(klass, *args, **kwargs): @@ -164,90 +161,6 @@ def get_awx_version(): return __version__ -def get_encryption_key(field_name, pk=None): - ''' - Generate key for encrypted password based on field name, - ``settings.SECRET_KEY``, and instance pk (if available). - - :param pk: (optional) the primary key of the ``awx.conf.model.Setting``; - can be omitted in situations where you're encrypting a setting - that is not database-persistent (like a read-only setting) - ''' - from django.conf import settings - h = hashlib.sha1() - h.update(settings.SECRET_KEY) - if pk is not None: - h.update(str(pk)) - h.update(field_name) - return h.digest()[:16] - - -def encrypt_field(instance, field_name, ask=False, subfield=None, skip_utf8=False): - ''' - Return content of the given instance and field name encrypted. - ''' - value = getattr(instance, field_name) - if isinstance(value, dict) and subfield is not None: - value = value[subfield] - if not value or value.startswith('$encrypted$') or (ask and value == 'ASK'): - return value - if skip_utf8: - utf8 = False - else: - utf8 = type(value) == six.text_type - value = smart_str(value) - key = get_encryption_key(field_name, getattr(instance, 'pk', None)) - cipher = AES.new(key, AES.MODE_ECB) - while len(value) % cipher.block_size != 0: - value += '\x00' - encrypted = cipher.encrypt(value) - b64data = base64.b64encode(encrypted) - tokens = ['$encrypted', 'AES', b64data] - if utf8: - # If the value to encrypt is utf-8, we need to add a marker so we - # know to decode the data when it's decrypted later - tokens.insert(1, 'UTF8') - return '$'.join(tokens) - - -def decrypt_value(encryption_key, value): - raw_data = value[len('$encrypted$'):] - # If the encrypted string contains a UTF8 marker, discard it - utf8 = raw_data.startswith('UTF8$') - if utf8: - raw_data = raw_data[len('UTF8$'):] - algo, b64data = raw_data.split('$', 1) - if algo != 'AES': - raise ValueError('unsupported algorithm: %s' % algo) - encrypted = base64.b64decode(b64data) - cipher = AES.new(encryption_key, AES.MODE_ECB) - value = cipher.decrypt(encrypted) - value = value.rstrip('\x00') - # If the encrypted string contained a UTF8 marker, decode the data - if utf8: - value = value.decode('utf-8') - return value - - -def decrypt_field(instance, field_name, subfield=None): - ''' - Return content of the given instance and field name decrypted. - ''' - value = getattr(instance, field_name) - if isinstance(value, dict) and subfield is not None: - value = value[subfield] - if not value or not value.startswith('$encrypted$'): - return value - key = get_encryption_key(field_name, getattr(instance, 'pk', None)) - - return decrypt_value(key, value) - - -def decrypt_field_value(pk, field_name, value): - key = get_encryption_key(field_name, pk) - return decrypt_value(key, value) - - def update_scm_url(scm_type, url, username=True, password=True, check_special_cases=True, scp_format=False): ''' diff --git a/awx/main/utils/encryption.py b/awx/main/utils/encryption.py new file mode 100644 index 0000000000..abdf8da5fd --- /dev/null +++ b/awx/main/utils/encryption.py @@ -0,0 +1,86 @@ +import base64 +import hashlib + +import six +from cryptography.fernet import Fernet + +from django.utils.encoding import smart_str + + +__all__ = ['get_encryption_key', 'encrypt_field', 'decrypt_field', 'decrypt_value'] + + +def get_encryption_key(field_name, pk=None): + ''' + Generate key for encrypted password based on field name, + ``settings.SECRET_KEY``, and instance pk (if available). + + :param pk: (optional) the primary key of the model object; + can be omitted in situations where you're encrypting a setting + that is not database-persistent (like a read-only setting) + ''' + from django.conf import settings + h = hashlib.sha256() + h.update(settings.SECRET_KEY) + if pk is not None: + h.update(str(pk)) + h.update(field_name) + return base64.b64encode(h.digest()) + + +def encrypt_field(instance, field_name, ask=False, subfield=None, skip_utf8=False): + ''' + Return content of the given instance and field name encrypted. + ''' + value = getattr(instance, field_name) + if isinstance(value, dict) and subfield is not None: + value = value[subfield] + if not value or value.startswith('$encrypted$') or (ask and value == 'ASK'): + return value + if skip_utf8: + utf8 = False + else: + utf8 = type(value) == six.text_type + value = smart_str(value) + key = get_encryption_key(field_name, getattr(instance, 'pk', None)) + f = Fernet(key) + encrypted = f.encrypt(value) + b64data = base64.b64encode(encrypted) + tokens = ['$encrypted', 'AESCBC', b64data] + if utf8: + # If the value to encrypt is utf-8, we need to add a marker so we + # know to decode the data when it's decrypted later + tokens.insert(1, 'UTF8') + return '$'.join(tokens) + + +def decrypt_value(encryption_key, value): + raw_data = value[len('$encrypted$'):] + # If the encrypted string contains a UTF8 marker, discard it + utf8 = raw_data.startswith('UTF8$') + if utf8: + raw_data = raw_data[len('UTF8$'):] + algo, b64data = raw_data.split('$', 1) + if algo != 'AESCBC': + raise ValueError('unsupported algorithm: %s' % algo) + encrypted = base64.b64decode(b64data) + f = Fernet(encryption_key) + value = f.decrypt(encrypted) + # If the encrypted string contained a UTF8 marker, decode the data + if utf8: + value = value.decode('utf-8') + return value + + +def decrypt_field(instance, field_name, subfield=None): + ''' + Return content of the given instance and field name decrypted. + ''' + value = getattr(instance, field_name) + if isinstance(value, dict) and subfield is not None: + value = value[subfield] + if not value or not value.startswith('$encrypted$'): + return value + key = get_encryption_key(field_name, getattr(instance, 'pk', None)) + + return decrypt_value(key, value) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index f1f40aef22..ba131f7b59 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -34,3 +34,5 @@ * Fixed an issue installing Tower on multiple nodes where cluster internal node references are used [[#6231](https://github.com/ansible/ansible-tower/pull/6231)] +* Tower now uses [Fernet](https://github.com/fernet/spec/blob/master/Spec.md) *(AESCBC w/ SHA256 HMAC)* + for all encrypted fields. [[#826](https://github.com/ansible/ansible-tower/pull/6541)] |