tests: Make it possible to run all tests using our infrastructure.

* Makefile.am (TESTS_ENVIRONMENT): New variable.
(check-all): New phony target to run all tests.
* tests/gpgme/gpgme-defs.scm (have-gpgme?): New function that tests
whether the GPGME test suite is available instead of exiting the
process.
* tests/gpgscm/init.scm (export): New macro.
* tests/gpgscm/tests.scm (run-tests): New function.
(load-tests): Likewise.
* tests/gpgme/run-tests.scm: Simplify and move the parsing of the list
of tests to 'all-tests.scm'.
* tests/gpgsm/run-tests.scm: Likewise.
* tests/migrations/run-tests.scm: Likewise.
* tests/openpgp/run-tests.scm: Likewise.
* tests/gpgme/Makefile.am: To select the tests to run, use the
variable 'TESTS'.  This harmonizes the interface with the automake
test suite.
* tests/gpgsm/Makefile.am: Likewise.
* tests/migrations/Makefile.am: Likewise.
* tests/openpgp/Makefile.am: Likewise.
* tests/openpgp/README: Likewise.
* agent/all-tests.scm: New file.
* common/all-tests.scm: Likewise.
* g10/all-tests.scm: Likewise.
* g13/all-tests.scm: Likewise.
* tests/gpgme/all-tests.scm: Likewise.
* tests/gpgsm/all-tests.scm: Likewise.
* tests/migrations/all-tests.scm: Likewise.
* tests/openpgp/all-tests.scm: Likewise.
* tests/run-tests.scm: Likewise.
--

This change allows us to run all tests in parallel and write one XML
report capturing the results of every test.  It also lays the
foundation to parametrize test suites.

Signed-off-by: Justus Winter <justus@g10code.com>

3 files changed, 59 insertions, 15 deletions

diff --git a/tests/gpgsm/Makefile.am b/tests/gpgsm/Makefile.am
index 892d3bc8b..e54db7842 100644
--- a/tests/gpgsm/Makefile.am
+++ b/tests/gpgsm/Makefile.am
@@ -54,7 +54,7 @@ check: xcheck
 .PHONY: xcheck
 xcheck:
 	$(TESTS_ENVIRONMENT) $(abs_top_builddir)/tests/gpgscm/gpgscm \
-	  $(abs_srcdir)/run-tests.scm $(TESTFLAGS) $(XTESTS)
+	  $(abs_srcdir)/run-tests.scm $(TESTFLAGS) $(TESTS)
 
 KEYS =	32100C27173EF6E9C4E9A25D3D69F86D37A4F939
 CERTS =	cert_g10code_test1.der \
@@ -66,7 +66,7 @@ TEST_FILES = plain-1.cms.asc \
 	plain-large.cms.asc
 
 EXTRA_DIST = $(XTESTS) $(KEYS) $(CERTS) $(TEST_FILES) \
-	gpgsm-defs.scm run-tests.scm setup.scm
+	gpgsm-defs.scm run-tests.scm setup.scm all-tests.scm
 
 CLEANFILES = *.log report.xml
 
diff --git a/tests/gpgsm/all-tests.scm b/tests/gpgsm/all-tests.scm
new file mode 100644
index 000000000..1baa92468
--- /dev/null
+++ b/tests/gpgsm/all-tests.scm
@@ -0,0 +1,43 @@
+;; Copyright (C) 2017 g10 Code GmbH
+;;
+;; This file is part of GnuPG.
+;;
+;; GnuPG is free software; you can redistribute it and/or modify
+;; it under the terms of the GNU General Public License as published by
+;; the Free Software Foundation; either version 3 of the License, or
+;; (at your option) any later version.
+;;
+;; GnuPG is distributed in the hope that it will be useful,
+;; but WITHOUT ANY WARRANTY; without even the implied warranty of
+;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;; GNU General Public License for more details.
+;;
+;; You should have received a copy of the GNU General Public License
+;; along with this program; if not, see <http://www.gnu.org/licenses/>.
+
+(export all-tests
+ ;; Parse the Makefile.am to find all tests.
+
+ (load (with-path "makefile.scm"))
+
+ (define (expander filename port key)
+   (parse-makefile port key))
+
+ (define (parse filename key)
+   (parse-makefile-expand filename expander key))
+
+ (define setup
+   (make-environment-cache
+    (test::scm
+     #f
+     (path-join "tests" "gpgsm" "setup.scm")
+     (in-srcdir "tests" "gpgsm" "setup.scm")
+     "--" "tests" "gpg")))
+
+ (map (lambda (name)
+	(test::scm setup
+		   (path-join "tests" "gpgsm" name)
+		   (in-srcdir "tests" "gpgsm" name)))
+      (parse-makefile-expand (in-srcdir "tests" "gpgsm" "Makefile.am")
+			     (lambda (filename port key) (parse-makefile port key))
+			     "XTESTS")))
diff --git a/tests/gpgsm/run-tests.scm b/tests/gpgsm/run-tests.scm
index bf129a1cb..6b460b165 100644
--- a/tests/gpgsm/run-tests.scm
+++ b/tests/gpgsm/run-tests.scm
@@ -23,16 +23,17 @@
 	    "tests/gpgsm.")
       (exit 2)))
 
-(let* ((tests (filter (lambda (arg) (not (string-prefix? arg "--"))) *args*))
-       (setup (make-environment-cache (test::scm
-				       #f
-				       (path-join "tests" "gpgsm" "setup.scm")
-				       (in-srcdir "tests" "gpgsm" "setup.scm"))))
-       (runner (if (and (member "--parallel" *args*)
-			(> (length tests) 1))
-		   run-tests-parallel
-		   run-tests-sequential)))
-  (runner (map (lambda (name)
-		 (test::scm setup
-			    (path-join "tests" "gpgsm" name)
-			    (in-srcdir "tests" "gpgsm" name))) tests)))
+(define tests (filter (lambda (arg) (not (string-prefix? arg "--"))) *args*))
+
+(define setup
+  (make-environment-cache (test::scm
+			   #f
+			   (path-join "tests" "gpgsm" "setup.scm")
+			   (in-srcdir "tests" "gpgsm" "setup.scm"))))
+
+(run-tests (if (null? tests)
+	       (load-tests "tests" "gpgsm")
+	       (map (lambda (name)
+		      (test::scm setup
+				 (path-join "tests" "gpgsm" name)
+				 (in-srcdir "tests" "gpgsm" name))) tests)))