Add Python Cryptography.io external test suite

Add python cryptography testing instructions too

Reviewed-by: Andy Polyakov <appro@openssl.org>
Reviewed-by: Richard Levitte <levitte@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2885)

5 files changed, 143 insertions, 11 deletions

diff --git a/.gitmodules b/.gitmodules
index 0800f20b70..696fdff396 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,3 +1,7 @@
 [submodule "boringssl"]
 	path = boringssl
 	url = https://boringssl.googlesource.com/boringssl
+
+[submodule "pyca.cryptography"]
+	path = pyca-cryptography
+	url = https://github.com/pyca/cryptography.git
diff --git a/pyca-cryptography b/pyca-cryptography
new file mode 160000
+Subproject 722235c46721acfe8b601e7846730c3c1fa588c
diff --git a/test/README.external b/test/README.external
index f411e731ec..f74b8a72b0 100644
--- a/test/README.external
+++ b/test/README.external
@@ -2,34 +2,38 @@ Running external test suites with OpenSSL
 =========================================
 
 It is possible to integrate external test suites into OpenSSL's "make test".
-This capability is considered a developer option and may not work on all
+This capability is considered a developer option and does not work on all
 platforms.
 
-At the current time the only supported external suite is the one used by
-BoringSSL.
 
 
 The BoringSSL test suite
 ========================
 
 In order to run the BoringSSL tests with OpenSSL, first checkout the BoringSSL
-source code into an appropriate directory:
+source code into an appropriate directory. This can be done in two ways:
 
-$ git clone https://boringssl.googlesource.com/boringssl boringssl
+1) Separately from the OpenSSL checkout using:
 
-The BoringSSL tests are only confirmed to work at a specific commit in the
-BoringSSL repository. Later commits may or may not pass the test suite:
+  $ git clone https://boringssl.googlesource.com/boringssl boringssl
 
-$ cd boringssl
-$ git checkout 490469f850e
+  The BoringSSL tests are only confirmed to work at a specific commit in the
+  BoringSSL repository. Later commits may or may not pass the test suite:
 
-From the OpenSSL source code configure to use the external tests:
+  $ cd boringssl
+  $ git checkout 490469f850e
+
+2) Using the already configured submodule settings in OpenSSL:
+
+  $ git submodule update --init
+
+Configure the OpenSSL source code to enable the external tests:
 
 $ cd ../openssl
 $ ./config enable-ssl3 enable-ssl3-method enable-weak-ssl-ciphers \
   enable-external-tests
 
-Note that using other config option than those given above may cause the tests
+Note that using other config options than those given above may cause the tests
 to fail.
 
 Run the OpenSSL tests by providing the path to the BoringSSL test runner in the
@@ -67,3 +71,30 @@ within the OpenSSL source code.
 The community is encouraged to contribute patches which reduce the number of
 suppressions that are currently present.
 
+
+Python PYCA/Cryptography test suite
+===================================
+
+This python test suite runs cryptographic tests with a local OpenSSL build as
+the implementation.
+
+First checkout the PYCA/Cryptography module into ./pyca-cryptography using:
+
+$ git submodule update --init
+
+Then configure/build OpenSSL compatible with the python module:
+
+$ ./config shared enable-external-tests
+$ make
+
+The tests will run in a python virtual environment which requires virtualenv
+to be installed.
+
+$ make test VERBOSE=1 TESTS=test_external_pyca
+
+Test failures and suppressions
+==============================
+
+Some tests target older (<=1.0.2) versions so will not run. Other tests target
+other crypto implementations so are not relevant. Currently no tests fail.
+
diff --git a/test/recipes/95-test_external_pyca.t b/test/recipes/95-test_external_pyca.t
new file mode 100644
index 0000000000..7ced77ea45
--- /dev/null
+++ b/test/recipes/95-test_external_pyca.t
@@ -0,0 +1,30 @@
+#! /usr/bin/env perl
+# Copyright 2015-2016 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the OpenSSL license (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+
+
+use OpenSSL::Test;
+use OpenSSL::Test::Utils;
+use OpenSSL::Test qw/:DEFAULT bldtop_file data_file srctop_file cmdstr/;
+
+setup("test_external");
+
+plan skip_all => "No external tests in this configuration"
+    if disabled("external-tests");
+
+plan tests => 1;
+
+SKIP: {
+    skip "PYCA Cryptography not available", 1
+        if ! -f srctop_file("pyca-cryptography", "setup.py");
+    skip "PYCA tests not available on Windows or VMS", 1
+        if $^O =~ /^(VMS|MSWin32)$/;
+
+    ok(run(cmd(["sh", data_file("cryptography.sh")])),
+        "running Python Cryptography tests");
+}
+
diff --git a/test/recipes/95-test_external_pyca_data/cryptography.sh b/test/recipes/95-test_external_pyca_data/cryptography.sh
new file mode 100755
index 0000000000..34d1a8a5be
--- /dev/null
+++ b/test/recipes/95-test_external_pyca_data/cryptography.sh
@@ -0,0 +1,67 @@
+#!/bin/sh
+#
+# Copyright 2017 The OpenSSL Project Authors. All Rights Reserved.
+#
+# Licensed under the OpenSSL license (the "License").  You may not use
+# this file except in compliance with the License.  You can obtain a copy
+# in the file LICENSE in the source distribution or at
+# https://www.openssl.org/source/license.html
+#
+# ====================================================================
+# Copyright (c) 2017 Oracle and/or its affiliates.  All rights reserved.
+#
+
+#
+# OpenSSL external testing using the Python Cryptography module
+#
+set -e
+
+O_EXE=`pwd`/$BLDTOP/apps
+O_BINC=`pwd`/$BLDTOP/include
+O_SINC=`pwd`/$SRCTOP/include
+O_LIB=`pwd`/$BLDTOP
+
+export PATH=$O_EXE:$PATH
+export LD_LIBRARY_PATH=$O_LIB:$LD_LIBRARY_PATH
+
+# Check/Set openssl version
+OPENSSL_VERSION=`openssl version | cut -f 2 -d ' '`
+
+echo "------------------------------------------------------------------"
+echo "Testing OpenSSL using Python Cryptography:"
+echo "   CWD:                $PWD"
+echo "   SRCTOP:             $SRCTOP"
+echo "   BLDTOP:             $BLDTOP"
+echo "   OpenSSL version:    $OPENSSL_VERSION"
+echo "------------------------------------------------------------------"
+
+cd $SRCTOP
+
+# Create a python virtual env and activate
+rm -rf venv-pycrypto
+virtualenv venv-pycrypto
+. ./venv-pycrypto/bin/activate
+
+cd pyca-cryptography
+
+pip install -q --requirement dev-requirements.txt
+
+echo "------------------------------------------------------------------"
+echo "Building cryptography"
+echo "------------------------------------------------------------------"
+python ./setup.py clean
+
+CFLAGS="-I$O_BINC -I$O_SINC -L$O_LIB" python ./setup.py build
+
+echo "------------------------------------------------------------------"
+echo "Running tests"
+echo "------------------------------------------------------------------"
+
+CFLAGS="-I$O_BINC -I$O_SINC -L$O_LIB" python ./setup.py test
+
+cd ../
+deactivate
+rm -rf venv-pycrypto
+
+exit 0
+