test: rework how images are created

Before, we'd create a separate image for each test, in
/var/tmp/systemd-test.XXXXX/rootdisk.img. Most of the images
where very similar, except that each one had some unit files installed
specifically for the test. The installation of those custom unit files
was removed in previous commits (all the unit files are always installed).

The new approach is to only create as few distinct images as possible.
We have:
default.img: the "normal" image suitable for almost all the tests
basic.img: the same as default image but doesn't mask any services
cryptsetup.img: p2 is used for encrypted /var
badid.img: /etc/machine-id is overwritten with stuff
selinux.img: with selinux added for fun and fun
and a few others:

ls -l build/test/*img
lrwxrwxrwx 1 root root 38 Mar 21 21:23 build/test/badid.img -> /var/tmp/systemd-test.PJFFeo/badid.img
lrwxrwxrwx 1 root root 38 Mar 21 21:17 build/test/basic.img -> /var/tmp/systemd-test.na0xOI/basic.img
lrwxrwxrwx 1 root root 43 Mar 21 21:18 build/test/cryptsetup.img -> /var/tmp/systemd-test.Tzjv06/cryptsetup.img
lrwxrwxrwx 1 root root 40 Mar 21 21:19 build/test/default.img -> /var/tmp/systemd-test.EscAsS/default.img
lrwxrwxrwx 1 root root 39 Mar 21 21:22 build/test/nspawn.img -> /var/tmp/systemd-test.HSebKo/nspawn.img
lrwxrwxrwx 1 root root 40 Mar 21 21:20 build/test/selinux.img -> /var/tmp/systemd-test.daBjbx/selinux.img
lrwxrwxrwx 1 root root 39 Mar 21 21:21 build/test/test08.img -> /var/tmp/systemd-test.OgnN8Z/test08.img

I considered trying to use the same image everywhere. It would probably be
possible, but it would be very brittle. By using separate images where it is
necessary we keep various orthogonal modifications independent.

The way that images are cached is complicated by the fact that we still
want to keep them in /var/tmp. Thus, an image is created on first use and
linked to from build/test/ so it can be found by other tests.

Tests cannot be run in parallel. I think that is an acceptable limitation.
Creation of the images was probably taking more resources then the actual
tests, so we should be better off anyway.

10 files changed, 113 insertions, 37 deletions

diff --git a/test/TEST-01-BASIC/test.sh b/test/TEST-01-BASIC/test.sh
index c3901e2f57..9f2ec1e8ae 100755
--- a/test/TEST-01-BASIC/test.sh
+++ b/test/TEST-01-BASIC/test.sh
@@ -1,11 +1,12 @@
 #!/usr/bin/env bash
 set -e
 TEST_DESCRIPTION="Basic systemd setup"
+IMAGE_NAME="basic"
 RUN_IN_UNPRIVILEGED_CONTAINER=${RUN_IN_UNPRIVILEGED_CONTAINER:-yes}
 
 . $TEST_BASE_DIR/test-functions
 
-test_setup() {
+test_create_image() {
     create_empty_image_rootdir
 
     # Create what will eventually be our root filesystem onto an overlay
diff --git a/test/TEST-02-CRYPTSETUP/test.sh b/test/TEST-02-CRYPTSETUP/test.sh
index 2afc6f8ed4..3e3414b638 100755
--- a/test/TEST-02-CRYPTSETUP/test.sh
+++ b/test/TEST-02-CRYPTSETUP/test.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 set -e
 TEST_DESCRIPTION="cryptsetup systemd setup"
+IMAGE_NAME="cryptsetup"
 TEST_NO_NSPAWN=1
 
 . $TEST_BASE_DIR/test-functions
@@ -23,8 +24,7 @@ check_result_qemu() {
     return $ret
 }
 
-
-test_setup() {
+test_create_image() {
     create_empty_image_rootdir
     echo -n test >$TESTDIR/keyfile
     cryptsetup -q luksFormat --pbkdf pbkdf2 --pbkdf-force-iterations 1000 ${LOOPDEV}p2 $TESTDIR/keyfile
diff --git a/test/TEST-06-SELINUX/test.sh b/test/TEST-06-SELINUX/test.sh
index f17242c6e1..7a836bbc03 100755
--- a/test/TEST-06-SELINUX/test.sh
+++ b/test/TEST-06-SELINUX/test.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 set -e
 TEST_DESCRIPTION="SELinux tests"
+IMAGE_NAME="selinux"
 TEST_NO_NSPAWN=1
 
 # Requirements:
@@ -15,7 +16,7 @@ test -f /usr/share/selinux/devel/include/system/systemd.if || exit 0
 SETUP_SELINUX=yes
 KERNEL_APPEND="$KERNEL_APPEND selinux=1 security=selinux"
 
-test_setup() {
+test_create_image() {
     create_empty_image_rootdir
 
     # Create what will eventually be our root filesystem onto an overlay
diff --git a/test/TEST-08-ISSUE-2730/test.sh b/test/TEST-08-ISSUE-2730/test.sh
index 80f3efad62..e5dedf2f0c 100755
--- a/test/TEST-08-ISSUE-2730/test.sh
+++ b/test/TEST-08-ISSUE-2730/test.sh
@@ -1,13 +1,14 @@
 #!/usr/bin/env bash
 set -e
 TEST_DESCRIPTION="https://github.com/systemd/systemd/issues/2730"
+IMAGE_NAME="test08"
 TEST_NO_NSPAWN=1
 
 . $TEST_BASE_DIR/test-functions
 QEMU_TIMEOUT=300
 FSTYPE=ext4
 
-test_setup() {
+test_create_image() {
     create_empty_image_rootdir
 
     # Create what will eventually be our root filesystem onto an overlay
diff --git a/test/TEST-13-NSPAWN-SMOKE/test.sh b/test/TEST-13-NSPAWN-SMOKE/test.sh
index 48449c66e5..c777c166f7 100755
--- a/test/TEST-13-NSPAWN-SMOKE/test.sh
+++ b/test/TEST-13-NSPAWN-SMOKE/test.sh
@@ -1,11 +1,12 @@
 #!/usr/bin/env bash
 set -e
 TEST_DESCRIPTION="systemd-nspawn smoke test"
+IMAGE_NAME=nspawn
 TEST_NO_NSPAWN=1
 
 . $TEST_BASE_DIR/test-functions
 
-test_setup() {
+test_create_image() {
     create_empty_image_rootdir
 
     # Create what will eventually be our root filesystem onto an overlay
diff --git a/test/TEST-14-MACHINE-ID/test.sh b/test/TEST-14-MACHINE-ID/test.sh
index 4668392b91..d1486f0aae 100755
--- a/test/TEST-14-MACHINE-ID/test.sh
+++ b/test/TEST-14-MACHINE-ID/test.sh
@@ -1,11 +1,12 @@
 #!/usr/bin/env bash
 set -e
 TEST_DESCRIPTION="/etc/machine-id testing"
+IMAGE_NAME=badid
 TEST_NO_NSPAWN=1
 
 . $TEST_BASE_DIR/test-functions
 
-test_setup() {
+test_create_image() {
     create_empty_image_rootdir
 
     # Create what will eventually be our root filesystem onto an overlay
diff --git a/test/TEST-21-SYSUSERS/test.sh b/test/TEST-21-SYSUSERS/test.sh
index c4b221af8a..f7dbbbf47b 100755
--- a/test/TEST-21-SYSUSERS/test.sh
+++ b/test/TEST-21-SYSUSERS/test.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -e
 TEST_DESCRIPTION="Sysuser-related tests"
-
+IMAGE_NAME=sysusers
 . $TEST_BASE_DIR/test-functions
 
 test_setup() {
diff --git a/test/TEST-24-UNIT-TESTS/test.sh b/test/TEST-24-UNIT-TESTS/test.sh
index 06b6cebf4f..0475621623 100755
--- a/test/TEST-24-UNIT-TESTS/test.sh
+++ b/test/TEST-24-UNIT-TESTS/test.sh
@@ -24,13 +24,13 @@ check_result_nspawn() {
     fi
     cp -a $TESTDIR/$1/var/log/journal $TESTDIR
     [[ -n "$TIMED_OUT" ]] && _ret=$(($_ret+1))
+    umount_initdir
     return $_ret
 }
 
 check_result_qemu() {
     local _ret=1
-    mkdir -p $initdir
-    mount ${LOOPDEV}p1 $initdir
+    mount_initdir
     [[ -e $initdir/testok ]] && _ret=0
     if [[ -s $initdir/failed ]]; then
         _ret=$(($_ret+1))
@@ -47,7 +47,7 @@ check_result_qemu() {
         fi
     fi
     cp -a $initdir/var/log/journal $TESTDIR
-    umount $initdir
+    umount_initdir
     [[ -n "$TIMED_OUT" ]] && _ret=$(($_ret+1))
     return $_ret
 }
diff --git a/test/run-integration-tests.sh b/test/run-integration-tests.sh
index c0a8448a88..19b5fe0315 100755
--- a/test/run-integration-tests.sh
+++ b/test/run-integration-tests.sh
@@ -4,8 +4,10 @@ set -e
 BUILD_DIR="$($(dirname "$0")/../tools/find-build-dir.sh)"
 if [ $# -gt 0 ]; then
     args="$@"
+    do_clean=0
 else
-    args="clean setup run clean-again"
+    args="setup run"
+    do_clean=1
 fi
 
 ninja -C "$BUILD_DIR"
@@ -16,6 +18,13 @@ COUNT=0
 FAILURES=0
 
 cd "$(dirname "$0")"
+
+if [ $do_clean = 1 ]; then
+    for TEST in TEST-??-* ; do
+        ( set -x ; make -C "$TEST" "BUILD_DIR=$BUILD_DIR" clean )
+    done
+fi
+
 for TEST in TEST-??-* ; do
     COUNT=$(($COUNT+1))
 
@@ -31,6 +40,12 @@ for TEST in TEST-??-* ; do
     [ "$RESULT" -ne "0" ] && FAILURES=$(($FAILURES+1))
 done
 
+if [ $FAILURES -eq 0 -a $do_clean = 1 ]; then
+    for TEST in TEST-??-* ; do
+        ( set -x ; make -C "$TEST" "BUILD_DIR=$BUILD_DIR" clean-again )
+    done
+fi
+
 echo ""
 
 for TEST in ${!results[@]}; do
diff --git a/test/test-functions b/test/test-functions
index 0e3c2734e2..bca0588af8 100644
--- a/test/test-functions
+++ b/test/test-functions
@@ -16,6 +16,7 @@ TIMED_OUT=  # will be 1 after run_* if *_TIMEOUT is set and test timed out
 UNIFIED_CGROUP_HIERARCHY="${UNIFIED_CGROUP_HIERARCHY:-default}"
 EFI_MOUNT="${EFI_MOUNT:-$(bootctl -x 2>/dev/null || echo /boot)}"
 QEMU_MEM="${QEMU_MEM:-512M}"
+IMAGE_NAME=${IMAGE_NAME:-default}
 
 # Decide if we can (and want to) run QEMU with KVM acceleration.
 # Check if nested KVM is explicitly enabled (TEST_NESTED_KVM). If not,
@@ -136,6 +137,7 @@ DEBUGTOOLS=(
 
 STATEDIR="${BUILD_DIR:-.}/test/$(basename $(dirname $(realpath $0)))"
 STATEFILE="$STATEDIR/.testdir"
+IMAGESTATEDIR="$STATEDIR/.."
 TESTLOG="$STATEDIR/test.log"
 
 is_built_with_asan() {
@@ -276,6 +278,9 @@ run_qemu() {
 
     find_qemu_bin || return 1
 
+    # Umount initdir to avoid concurrent access to the filesystem
+    umount_initdir
+
     local _cgroup_args
     if [[ "$UNIFIED_CGROUP_HIERARCHY" = "yes" ]]; then
         _cgroup_args="systemd.unified_cgroup_hierarchy=yes"
@@ -320,7 +325,7 @@ $KERNEL_APPEND \
 -m $QEMU_MEM \
 -nographic \
 -kernel $KERNEL_BIN \
--drive format=raw,cache=unsafe,file=${TESTDIR}/rootdisk.img \
+-drive format=raw,cache=unsafe,file=${IMAGESTATEDIR}/${IMAGE_NAME}.img \
 $QEMU_OPTIONS \
 "
 
@@ -631,7 +636,7 @@ install_systemd() {
     # and it could fill the available space
     strip_binaries
 
-   [[ "$LOOKS_LIKE_SUSE" ]] && setup_suse
+    [[ "$LOOKS_LIKE_SUSE" ]] && setup_suse
 
     # enable debug logging in PID1
     echo LogLevel=debug >> $initdir/etc/systemd/system.conf
@@ -658,19 +663,26 @@ install_missing_libraries() {
 }
 
 create_empty_image() {
+    if [ -z "$IMAGE_NAME" ]; then
+        echo "create_empty_image: \$IMAGE_NAME not set"
+        exit 1
+    fi
+
     local _size=500
     if [[ "$STRIP_BINARIES" = "no" ]]; then
         _size=$((4*_size))
     fi
 
-    echo "Setting up $TESTDIR/rootdisk.img (${_size} MB)"
+    image="${TESTDIR}/${IMAGE_NAME}.img"
+    public="$IMAGESTATEDIR/${IMAGE_NAME}.img"
+    echo "Setting up $public (${_size} MB)"
 
-    rm -f "$TESTDIR/rootdisk.img"
+    rm -f "$image" "$public"
     # Create the blank file to use as a root filesystem
-    truncate -s "${_size}M" "$TESTDIR/rootdisk.img"
-    LOOPDEV=$(losetup --show -P -f $TESTDIR/rootdisk.img)
+    truncate -s "${_size}M" "$image"
+    LOOPDEV=$(losetup --show -P -f "$image")
     [ -b "$LOOPDEV" ] || return 1
-    echo "LOOPDEV=$LOOPDEV" >> $STATEFILE
+    echo "LOOPDEV=$LOOPDEV" >>$STATEFILE
     sfdisk "$LOOPDEV" <<EOF
 ,$((_size-50))M
 ,
@@ -685,15 +697,39 @@ EOF
         dfatal "Failed to mkfs -t ${FSTYPE}"
         exit 1
     fi
+
+    # the image is created, let's expose it
+    ln -vs "$(realpath $image)" "$public"
 }
 
-create_empty_image_rootdir() {
-    create_empty_image
+mount_initdir() {
+    if [ -z "${LOOPDEV}" ]; then
+        image="${TESTDIR}/${IMAGE_NAME}.img"
+        LOOPDEV=$(losetup --show -P -f "$image")
+        [ -b "$LOOPDEV" ] || return 1
+        echo "LOOPDEV=$LOOPDEV" >>$STATEFILE
+    fi
+
     mkdir -p $initdir
     mount ${LOOPDEV}p1 $initdir
     TEST_SETUP_CLEANUP_ROOTDIR=1
 }
 
+umount_initdir() {
+    _umount_dir $initdir
+    if [[ $LOOPDEV && -b $LOOPDEV ]]; then
+        ddebug "losetup -d $LOOPDEV"
+        losetup -d $LOOPDEV
+    fi
+    LOOPDEV=
+    sed -i /LOOPDEV=/d $STATEFILE
+}
+
+create_empty_image_rootdir() {
+    create_empty_image
+    mount_initdir
+}
+
 check_asan_reports() {
     local ret=0
     local root="$1"
@@ -744,14 +780,14 @@ check_result_nspawn() {
     test -s $TESTDIR/failed && ret=$(($ret+1))
     [ -n "$TIMED_OUT" ] && ret=$(($ret+1))
     check_asan_reports "$TESTDIR/$1" || ret=$(($ret+1))
+    umount_initdir
     return $ret
 }
 
 # can be overridden in specific test
 check_result_qemu() {
     local ret=1
-    mkdir -p $initdir
-    mount ${LOOPDEV}p1 $initdir
+    mount_initdir
     [[ -e $initdir/testok ]] && ret=0
     [[ -f $initdir/failed ]] && cp -a $initdir/failed $TESTDIR
     cp -a $initdir/var/log/journal $TESTDIR
@@ -1042,6 +1078,10 @@ has_user_dbus_socket() {
 }
 
 setup_nspawn_root() {
+    if [ -z "${initdir}" ]; then
+        dfatal "\$initdir not defined"
+        exit 1
+    fi
     rm -fr $TESTDIR/nspawn-root
     ddebug "cp -ar $initdir $TESTDIR/nspawn-root"
     cp -ar $initdir $TESTDIR/nspawn-root
@@ -1110,7 +1150,10 @@ import_testdir() {
             mkdir -p "$TESTDIR"
         fi
 
-        echo "TESTDIR=\"$TESTDIR\"" > $STATEFILE
+        cat >$STATEFILE<<EOF
+TESTDIR="$TESTDIR"
+LOOPDEV="$LOOPDEV"
+EOF
         export TESTDIR
     fi
 }
@@ -1866,13 +1909,10 @@ _test_cleanup() {
     # (post-test) cleanup should always ignore failure and cleanup as much as possible
     (
         set +e
-        _umount_dir $initdir
-        if [[ $LOOPDEV && -b $LOOPDEV ]]; then
-            ddebug "losetup -d $LOOPDEV"
-            losetup -d $LOOPDEV
-        fi
-        rm -fr "$TESTDIR"
-        rm -f "$STATEFILE"
+        umount_initdir
+        rm -vf "${IMAGESTATEDIR}/${IMAGE_NAME}.img"
+        rm -vfr "$TESTDIR"
+        rm -vf "$STATEFILE"
     ) || :
 }
 
@@ -1881,12 +1921,7 @@ test_cleanup() {
     _test_cleanup
 }
 
-test_setup() {
-    if type -P meson >/dev/null && [[ "$(meson configure $BUILD_DIR | grep install-tests | awk '{ print $2 }')" != "true" ]]; then
-        dfatal "Needs to be built with -Dinstall-tests=true"
-        exit 1
-    fi
-
+test_create_image() {
     create_empty_image_rootdir
 
     # Create what will eventually be our root filesystem onto an overlay
@@ -1895,6 +1930,27 @@ test_setup() {
         setup_basic_environment
         mask_supporting_services
     )
+}
+
+test_setup() {
+    if type -P meson >/dev/null && [[ "$(meson configure $BUILD_DIR | grep install-tests | awk '{ print $2 }')" != "true" ]]; then
+        dfatal "Needs to be built with -Dinstall-tests=true"
+        exit 1
+    fi
+
+    image="${TESTDIR}/${IMAGE_NAME}.img"
+    public="${IMAGESTATEDIR}/${IMAGE_NAME}.img"
+    if [ -e "$image" ]; then
+        echo "Reusing existing image $PWD/$image → $(realpath $image)"
+        mount_initdir
+    elif [ -e "$public" ]; then
+        echo "Reusing existing cached image $PWD/$public → $(realpath $public)"
+        ln -s "$(realpath $public)" "$image"
+        mount_initdir
+    else
+        test_create_image
+    fi
+
     setup_nspawn_root
 }