From 7205fc7dc31eb2be3075ee6ba23ebe84324aa5cb Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 12 Jul 2024 15:33:49 +0200 Subject: mkosi: Introduce build image We want the exitrd image to be built with the latest systemd as well. As the exitrd image is built as part of mkosi.images, and all subimages are built before the main image, this implies the packages must be built as a subimage in mkosi.images/ as well. So we introduce the build image and move all logic related to building distribution packages there. This also has the nice side effect of slimming down the main image as the build dependencies are not installed into the main image anymore. It also makes sure the packages are built in a "clean" chroot without any of the other packages which we install in the main image available. --- mkosi.conf.d/10-arch/mkosi.build.chroot | 99 --------------- mkosi.conf.d/10-arch/mkosi.conf | 12 -- mkosi.conf.d/10-arch/mkosi.prepare | 42 ++++--- mkosi.conf.d/10-centos-fedora/mkosi.build.chroot | 126 ------------------- mkosi.conf.d/10-centos-fedora/mkosi.conf | 7 +- mkosi.conf.d/10-centos-fedora/mkosi.prepare | 66 ++-------- mkosi.conf.d/10-centos/mkosi.conf | 5 - mkosi.conf.d/10-debian-ubuntu/mkosi.build.chroot | 147 ----------------------- mkosi.conf.d/10-debian-ubuntu/mkosi.conf | 17 +-- mkosi.conf.d/10-debian-ubuntu/mkosi.postinst | 29 ----- mkosi.conf.d/10-debian-ubuntu/mkosi.prepare | 18 ++- mkosi.conf.d/10-fedora/mkosi.conf | 5 - mkosi.conf.d/10-opensuse/mkosi.build.chroot | 145 ---------------------- mkosi.conf.d/10-opensuse/mkosi.conf | 24 +--- mkosi.conf.d/10-opensuse/mkosi.prepare | 73 +++-------- mkosi.conf.d/20-build.conf | 9 ++ mkosi.conf.d/20-none.conf | 4 +- mkosi.conf.d/20-sanitizers.conf | 4 + 18 files changed, 78 insertions(+), 754 deletions(-) delete mode 100755 mkosi.conf.d/10-arch/mkosi.build.chroot delete mode 100755 mkosi.conf.d/10-centos-fedora/mkosi.build.chroot delete mode 100755 mkosi.conf.d/10-debian-ubuntu/mkosi.build.chroot delete mode 100755 mkosi.conf.d/10-debian-ubuntu/mkosi.postinst delete mode 100755 mkosi.conf.d/10-opensuse/mkosi.build.chroot create mode 100644 mkosi.conf.d/20-build.conf (limited to 'mkosi.conf.d') diff --git a/mkosi.conf.d/10-arch/mkosi.build.chroot b/mkosi.conf.d/10-arch/mkosi.build.chroot deleted file mode 100755 index 268bdc2ee7..0000000000 --- a/mkosi.conf.d/10-arch/mkosi.build.chroot +++ /dev/null @@ -1,99 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: LGPL-2.1-or-later -set -e - -if ((NO_BUILD)); then - exit 0 -fi - -# shellcheck source=/dev/null -. /usr/lib/os-release - -if [ ! -f "pkg/$ID/PKGBUILD" ]; then - echo "PKGBUILD not found at pkg/$ID/PKGBUILD, run mkosi once with -ff to make sure the PKGBUILD is cloned" >&2 - exit 1 -fi - -# We can't configure the source or build directory so we use symlinks instead to make sure they are in the -# expected locations. -ln --symbolic "$SRCDIR" "pkg/$ID/systemd" -ln --symbolic "$BUILDDIR" "pkg/$ID/build" -# Because we run with --noextract we are responsible for making sure the source files appear in src/. -ln --symbolic . "pkg/$ID/src" - -MKOSI_CFLAGS="-O0 -Wp,-U_FORTIFY_SOURCE" -if ((LLVM)); then - # TODO: Remove -fno-sanitize-function when https://github.com/systemd/systemd/issues/29972 is fixed. - MKOSI_CFLAGS="$MKOSI_CFLAGS -shared-libasan -fno-sanitize=function" -fi - -MKOSI_LDFLAGS="" -if ((LLVM)) && [[ -n "$SANITIZERS" ]]; then - MKOSI_LDFLAGS="$MKOSI_LDFLAGS -Wl,-rpath=$(clang --print-file-name="")lib/linux" -fi - -MKOSI_MESON_OPTIONS="-D mode=developer -D b_sanitize=${SANITIZERS:-none}" -if ((WIPE)) && [[ -d "$BUILDDIR/meson-private" ]]; then - MKOSI_MESON_OPTIONS="$MKOSI_MESON_OPTIONS --wipe" -fi - -# Override the default options. We specifically disable "strip", "zipman" and "lto" as they slow down builds -# significantly. OPTIONS= cannot be overridden on the makepkg command line so we append to /etc/makepkg.conf -# instead. The rootfs is overlaid with a writable tmpfs during the build script so these changes don't end up -# in the image itself. -tee --append /etc/makepkg.conf >/dev/null <&2 - exit 1 -fi - -# We get depends and optdepends from .SRCINFO as getting them from the PKGBUILD is rather complex. -sed --expression 's/^[ \t]*//' "pkg/$ID/.SRCINFO" | - grep --regexp '^depends =' --regexp '^optdepends =' | - sed --expression 's/^depends = //' --expression 's/^optdepends = //' --expression 's/:.*//' --expression 's/=.*//' | - xargs --delimiter '\n' mkosi-install +while read -r PACKAGE; do + DEPS="$DEPS $( + pacman --sync --info "$PACKAGE" | + sed '1,/^$/d' | # Only keep result from first repository (delete everything after first blank line). + sed --quiet 's/^Depends On *: //p' # Filter out everything except "Depends On:" line and fetch dependencies from it. + )" -# We get makedepends from the PKGBUILD as .SRCINFO can't encode conditional dependencies depending on -# whether some environment variable is set or not. -# shellcheck source=/dev/null -_systemd_UPSTREAM=1 . "pkg/$ID/PKGBUILD" + DEPS="$DEPS $( + pacman --sync --info "$PACKAGE" | + sed '1,/^$/d' | # Only keep result from first repository (delete everything after first blank line). + sed --quiet '/Optional Deps/,/Conflicts With/{/Conflicts With/!p}' | # Get every line from "Optional Deps" (inclusive) until "Conflicts With" (exclusive). + sed 's/Optional Deps *: //' | # Drop "Optional Deps :" from first line. + sed 's/ *\(.*\):.*/\1/' | # Drop descriptions (everything after first colon for all lines). + tr '\n' ' ' # Transform newlines to whitespace. + )" +done < <(jq --raw-output .VolatilePackages[] <"$MKOSI_CONFIG") -# shellcheck disable=SC2154 -mkosi-install "${makedepends[@]}" +echo "$DEPS" | + xargs | # Remove extra whitespace. + tr ' ' '\n' | + grep --invert-match --regexp systemd --regexp None | # systemd packages will be installed later on. + sort --unique | + xargs --delimiter '\n' --no-run-if-empty mkosi-install diff --git a/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot b/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot deleted file mode 100755 index dcb90c31cc..0000000000 --- a/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot +++ /dev/null @@ -1,126 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: LGPL-2.1-or-later -set -e - -. mkosi.functions - -if ((NO_BUILD)); then - exit 0 -fi - -# shellcheck source=/dev/null -. /usr/lib/os-release - -if [ ! -f "pkg/$ID/systemd.spec" ]; then - echo "spec not found at pkg/$ID/systemd.spec, run mkosi once with -ff to make sure the spec is cloned" >&2 - exit 1 -fi - -if [ -d .git/ ] && [ -z "$(git status --porcelain)" ]; then - TS="$(git show --no-patch --format=%ct HEAD)" -else - TS="${SOURCE_DATE_EPOCH:-$(date +%s)}" -fi - -if systemd-analyze compare-versions "$(rpm --version | cut -d ' ' -f3)" lt "4.19.91"; then - # Fix the %install override so debuginfo packages are generated even when --build-in-place is used. - # See https://github.com/rpm-software-management/rpm/issues/3042. - tee --append /usr/lib/rpm/redhat/macros <<'EOF' -%install %{?_enable_debug_packages:%{debug_package}}\ -%%install\ -%{nil} -EOF -fi - -VERSION="$(cat meson.version)" -RELEASE="$(date "+%Y%m%d%H%M%S" --date "@$TS")" - -DIST="$(rpm --eval %dist)" -ARCH="$(rpm --eval %_arch)" -SRCDEST="/usr/src/debug/systemd-$VERSION-${RELEASE}${DIST}.$ARCH" - -COMMON_MACRO_OVERRIDES=( - --define "toolchain $( ((LLVM)) && echo clang || echo gcc)" - --define "_fortify_level 0" - --undefine _lto_cflags - # TODO: Remove once redhat-rpm-config 292 is available everywhere. - --define "_hardening_clang_cflags --config=/usr/lib/rpm/redhat/redhat-hardened-clang.cfg" - --define "_hardening_clang_ldflags --config=/usr/lib/rpm/redhat/redhat-hardened-clang-ld.cfg" -) - -# TODO: Drop -U_FORTIFY_SOURCE when we switch to CentOS Stream 10. -MKOSI_CFLAGS="-O0 -Wp,-U_FORTIFY_SOURCE" -if ((WITH_DEBUG)); then - MKOSI_CFLAGS="$MKOSI_CFLAGS -fdebug-prefix-map=../src=$SRCDEST" -fi -if ((LLVM)); then - # TODO: Remove -fno-sanitize-function when https://github.com/systemd/systemd/issues/29972 is fixed. - MKOSI_CFLAGS="$MKOSI_CFLAGS -shared-libasan -fno-sanitize=function" -fi - -MKOSI_LDFLAGS="" -if ((LLVM)) && [[ -n "$SANITIZERS" ]]; then - MKOSI_LDFLAGS="$MKOSI_LDFLAGS -Wl,-rpath=$(dirname "$(clang --print-file-name=libclang_rt.asan.so)")" -fi - -MKOSI_MESON_OPTIONS="-D mode=developer -D b_sanitize=${SANITIZERS:-none}" -if ((WIPE)) && [[ -d "$BUILDDIR/meson-private" ]]; then - MKOSI_MESON_OPTIONS="$MKOSI_MESON_OPTIONS --wipe" -fi - -IFS= -# TODO: Replace meson_build and meson_install overrides with "--undefine __meson_verbose" once -# https://github.com/mesonbuild/meson/pull/12835 is available. -# shellcheck disable=SC2046 -env \ ---unset=CFLAGS \ ---unset=CXXFLAGS \ ---unset=LDFLAGS \ -ANNOBIN="no-active-checks" \ -CC_LD="$( ((LLVM)) && echo lld)" \ -CXX_LD="$( ((LLVM)) && echo lld)" \ - rpmbuild \ - -bb \ - --build-in-place \ - --with upstream \ - $( ((WITH_TESTS)) || echo "--nocheck") \ - $( ((WITH_DOCS)) || echo "--without=docs") \ - --define "_topdir /var/tmp" \ - --define "_sourcedir pkg/$ID" \ - --define "_rpmdir $OUTPUTDIR" \ - ${BUILDDIR:+"--define=_vpath_builddir $BUILDDIR"} \ - --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" \ - --define "_binary_payload w.ufdio" \ - $( ((WITH_DEBUG)) || echo "--define=debug_package %{nil}") \ - --define "version_override $VERSION" \ - --define "release_override $RELEASE" \ - "${COMMON_MACRO_OVERRIDES[@]}" \ - --define "build_cflags $(rpm "${COMMON_MACRO_OVERRIDES[@]}" --eval "%{?build_cflags}") $MKOSI_CFLAGS $CFLAGS" \ - --define "build_cxxflags $(rpm "${COMMON_MACRO_OVERRIDES[@]}" --eval "%{?build_cxxflags}") $MKOSI_CFLAGS $CFLAGS" \ - --define "build_ldflags $(rpm "${COMMON_MACRO_OVERRIDES[@]}" --eval "%{?build_ldflags}") $MKOSI_LDFLAGS $LDFLAGS" \ - --define "meson_build %{shrink:%{__meson} compile -C %{_vpath_builddir} -j %{_smp_build_ncpus} $( ((MESON_VERBOSE)) && echo --verbose) %{nil}}" \ - --define "meson_install %{shrink:DESTDIR=%{buildroot} %{__meson} install -C %{_vpath_builddir} --no-rebuild --quiet %{nil}}" \ - --define "meson_extra_configure_options $MKOSI_MESON_OPTIONS $MESON_OPTIONS" \ - $( ((WITH_DEBUG)) || echo "--define=__brp_strip %{nil}") \ - --define "__brp_compress %{nil}" \ - --define "__brp_mangle_shebangs %{nil}" \ - --define "__brp_strip_comment_note %{nil}" \ - --define "__brp_strip_static_archive %{nil}" \ - --define "__brp_check_rpaths %{nil}" \ - --define "__elf_exclude_path ^/usr/lib/systemd/tests/unit-tests/.*$" \ - --define "__script_requires %{nil}" \ - --define "_find_debuginfo_dwz_opts %{nil}" \ - --define "_fixperms true" \ - --undefine _package_note_flags \ - --noclean \ - "pkg/$ID/systemd.spec" - -( - shopt -s nullglob - rm -f "$BUILDDIR"/*.rpm -) - -cp "$OUTPUTDIR"/*.rpm "$PACKAGEDIR" -cp "$OUTPUTDIR"/*.rpm "$BUILDDIR" - -make_sysext_unsigned /var/tmp/BUILD/*/BUILDROOT diff --git a/mkosi.conf.d/10-centos-fedora/mkosi.conf b/mkosi.conf.d/10-centos-fedora/mkosi.conf index 4c6109a544..d046eb218b 100644 --- a/mkosi.conf.d/10-centos-fedora/mkosi.conf +++ b/mkosi.conf.d/10-centos-fedora/mkosi.conf @@ -11,6 +11,7 @@ VolatilePackages= systemd-container systemd-devel systemd-journal-remote + systemd-libs systemd-networkd systemd-networkd-defaults systemd-oomd-defaults @@ -23,7 +24,6 @@ VolatilePackages= Packages= bind-utils bpftool - compiler-rt cryptsetup device-mapper-event device-mapper-multipath @@ -33,7 +33,6 @@ Packages= git-core glibc-langpack-de glibc-langpack-en - gnutls gnutls-utils integritysetup iproute @@ -41,9 +40,7 @@ Packages= iputils iscsi-initiator-utils kernel-core - libasan libcap-ng-utils - libubsan man-db nmap-ncat openssh-clients @@ -57,8 +54,6 @@ Packages= python3-pexpect quota rpm - rpm-build - rpmautospec sbsigntools softhsm squashfs-tools diff --git a/mkosi.conf.d/10-centos-fedora/mkosi.prepare b/mkosi.conf.d/10-centos-fedora/mkosi.prepare index 1b86073ef5..2a890bcada 100755 --- a/mkosi.conf.d/10-centos-fedora/mkosi.prepare +++ b/mkosi.conf.d/10-centos-fedora/mkosi.prepare @@ -2,64 +2,18 @@ # SPDX-License-Identifier: LGPL-2.1-or-later set -e -if [ "$1" = "build" ] || ((NO_BUILD)); then +if [[ "$1" == "build" ]]; then exit 0 fi -# shellcheck source=/dev/null -. "$BUILDROOT/usr/lib/os-release" +mapfile -t PACKAGES < <(jq --raw-output .VolatilePackages[] <"$MKOSI_CONFIG") -if [ ! -f "pkg/$ID/systemd.spec" ]; then - echo "spec not found at pkg/$ID/systemd.spec, run mkosi with -ff to make sure the spec is cloned" >&2 - exit 1 -fi - -for DEPS in --requires --buildrequires; do - mkosi-chroot \ - rpmspec \ - --with upstream \ - --query \ - "$DEPS" \ - --define "_topdir /var/tmp" \ - --define "_sourcedir pkg/$ID" \ - "pkg/$ID/systemd.spec" | - grep --invert-match --regexp systemd --regexp /bin/sh --regexp "rpmlib(" --regexp udev --regexp grubby --regexp sdubby | - sort --unique | - tee /tmp/buildrequires | - xargs --delimiter '\n' mkosi-install -done - -# rpmbuild -br tries to build a source package which means all source files have to exist which isn't the -# case when using --build-in-place so we get rid of the source file that doesn't exist to make it happy. -# TODO: Use -bd instead of -br and get rid of this once we don't need to build on CentOS Stream 9 anymore. -sed '/Source0/d' --in-place "pkg/$ID/systemd.spec" - -until mkosi-chroot \ - rpmbuild \ - -br \ - --build-in-place \ - --with upstream \ - --define "_topdir /var/tmp" \ - --define "_sourcedir pkg/$ID" \ - --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" \ - "pkg/$ID/systemd.spec" -do - EXIT_STATUS=$? - if [ $EXIT_STATUS -ne 11 ]; then - exit $EXIT_STATUS - fi - - mkosi-chroot \ - rpm \ - --query \ - --package \ - --requires \ - /var/tmp/SRPMS/systemd-*.buildreqs.nosrc.rpm | - grep --invert-match '^rpmlib(' | - sort --unique >/tmp/dynamic-buildrequires - - sort /tmp/buildrequires /tmp/dynamic-buildrequires | - uniq --unique | - tee --append /tmp/buildrequires | - xargs --delimiter '\n' mkosi-install +for DEPS in --requires --recommends --suggests; do + # We need --latest-limit=1 to only consider the newest version of the packages. + # --latest-limit=1 is per . so we have to pass --arch= explicitly to make sure i686 packages + # are not considerd on x86-64. + dnf repoquery --arch="$DISTRIBUTION_ARCHITECTURE" --latest-limit=1 --quiet "$DEPS" "${PACKAGES[@]}" | + grep --invert-match --regexp systemd --regexp udev --regexp /bin/sh --regexp grubby --regexp sdubby --regexp libcurl-minimal | + sort --unique | + xargs --delimiter '\n' --no-run-if-empty mkosi-install done diff --git a/mkosi.conf.d/10-centos/mkosi.conf b/mkosi.conf.d/10-centos/mkosi.conf index 66aa0611d6..f9ea9a6596 100644 --- a/mkosi.conf.d/10-centos/mkosi.conf +++ b/mkosi.conf.d/10-centos/mkosi.conf @@ -20,10 +20,5 @@ Environment= # mkfs.ext4 enabled it by default, so we disable it explicitly. SYSTEMD_REPART_MKFS_OPTIONS_EXT4="-O ^orphan_file" - GIT_URL=https://git.centos.org/rpms/systemd.git - GIT_BRANCH=c9s-sig-hyperscale - GIT_COMMIT=8cf2aed0181920611421384f7374720db269d6c7 - Packages= kernel-modules # For squashfs - rpmautospec-rpm-macros diff --git a/mkosi.conf.d/10-debian-ubuntu/mkosi.build.chroot b/mkosi.conf.d/10-debian-ubuntu/mkosi.build.chroot deleted file mode 100755 index 1a03dcd956..0000000000 --- a/mkosi.conf.d/10-debian-ubuntu/mkosi.build.chroot +++ /dev/null @@ -1,147 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: LGPL-2.1-or-later -set -e - -if ((NO_BUILD)); then - exit 0 -fi - -# shellcheck source=/dev/null -. /usr/lib/os-release - -if [ ! -d "pkg/$ID/debian" ]; then - echo "deb rules not found at pkg/$ID/debian, run mkosi once with -ff to make sure the rules are cloned" >&2 - exit 1 -fi - -# We transplant the debian/ folder from the deb package sources into the upstream sources. -mount --mkdir --bind "$SRCDIR/pkg/$ID/debian" "$SRCDIR"/debian - -# We remove the patches so they don't get applied. -rm -rf "$SRCDIR"/debian/patches/* - -# While the build directory can be specified through DH_OPTIONS, the default one is hardcoded everywhere so -# we have to use that. Because it is architecture dependent, we query it using dpkg-architecture first. -DEB_HOST_GNU_TYPE="$(dpkg-architecture --query DEB_HOST_GNU_TYPE)" -mount --mkdir --bind "$BUILDDIR" "$SRCDIR/obj-$DEB_HOST_GNU_TYPE" - -if [ -d .git/ ] && [ -z "$(git status --porcelain)" ]; then - TS="$(git show --no-patch --format=%ct HEAD)" -else - TS="${SOURCE_DATE_EPOCH:-$(date +%s)}" -fi - -# Add a new changelog entry to update the version. We use a fixed date since a dynamic one causes a full -# rebuild every time. -cat >debian/changelog.new < $(date --rfc-email --date "@$TS") - -EOF -cat debian/changelog >>debian/changelog.new -mv debian/changelog.new debian/changelog - -MKOSI_CFLAGS="-O0" -if ((LLVM)); then - # TODO: Remove -fno-sanitize-function when https://github.com/systemd/systemd/issues/29972 is fixed. - MKOSI_CFLAGS="$MKOSI_CFLAGS -shared-libasan -fno-sanitize=function" -fi - -MKOSI_LDFLAGS="" -if ((LLVM)) && [[ -n "$SANITIZERS" ]]; then - MKOSI_LDFLAGS="$MKOSI_LDFLAGS -Wl,-rpath=$(clang --print-file-name="")lib/linux" -fi - -MKOSI_MESON_OPTIONS="-D mode=developer -D b_sanitize=${SANITIZERS:-none}" -if ((WIPE)) && [[ -d "$BUILDDIR/meson-private" ]]; then - MKOSI_MESON_OPTIONS="$MKOSI_MESON_OPTIONS --wipe" -fi - -# TODO: Drop GENSYMBOLS_LEVEL once https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=986746 is fixed. -build() { - env \ - CC="$( ((LLVM)) && echo clang || echo gcc)" \ - CXX="$( ((LLVM)) && echo clang++ || echo g++)" \ - CC_LD="$( ((LLVM)) && echo lld)" \ - CXX_LD="$( ((LLVM)) && echo lld)" \ - DEB_BUILD_OPTIONS="$(awk '$1=$1' <<<"\ - $( ((WITH_TESTS)) || echo nocheck) \ - $( ((WITH_DOCS)) || echo nodoc) \ - $( ((WITH_DEBUG)) && echo debug || echo nostrip) \ - $( ! ((MESON_VERBOSE)) && echo terse) \ - optimize=-lto \ - hardening=-fortify \ - ")" \ - DEB_BUILD_PROFILES="$(awk '$1=$1' <<<"\ - $( ((WITH_TESTS)) || echo nocheck) \ - $( ((WITH_DOCS)) || echo nodoc) \ - pkg.systemd.upstream \ - ")" \ - DEB_CFLAGS_APPEND="$MKOSI_CFLAGS $CFLAGS" \ - DEB_CXXFLAGS_APPEND="$MKOSI_CFLAGS $CFLAGS" \ - DEB_LDFLAGS_APPEND="$MKOSI_LDFLAGS $LDFLAGS" \ - DPKG_FORCE="unsafe-io" \ - DPKG_DEB_COMPRESSOR_TYPE="none" \ - DH_MISSING="--fail-missing" \ - CONFFLAGS_UPSTREAM="$MKOSI_MESON_OPTIONS $MESON_OPTIONS" \ - GENSYMBOLS_LEVEL="$( ((LLVM)) && echo 0 || echo 1)" \ - dpkg-buildpackage \ - --no-pre-clean \ - --unsigned-changes \ - --build=binary - - EXIT_STATUS=$? - - # Make sure we don't reconfigure twice. - MKOSI_MESON_OPTIONS="${MKOSI_MESON_OPTIONS//"--wipe"/}" - - return $EXIT_STATUS -} - -if ! build; then - # debhelper installs files for each package to debian/ so we figure out which files were - # packaged by querying all the package names from debian/control and running find on each of the - # corresponding package directory in debian/. - grep "Package:" debian/control | - sed "s/Package: //" | - xargs -d '\n' -I {} sh -c "[ -d debian/{} ] && (cd debian/{} && find . ! -type d ! -path "*dh-exec*" -printf '%P\n')" | - # Remove compression suffix from compressed manpages as the manpages in debian/tmp will be uncompressed. - sed --regexp-extended 's/([0-9])\.gz$/\1/' | - sort --unique >/tmp/packaged-files - - # We figure out the installed files by running find on debian/tmp/ which contains the files installed - # by meson install. - (cd debian/tmp/ && find . ! -type d ! -path "*dh-exec*" -printf '%P\n') >/tmp/installed-files - - if [ -f debian/not-installed ]; then - grep --invert-match "^#" debian/not-installed >>/tmp/installed-files - fi - - sort --unique --output /tmp/installed-files /tmp/installed-files - - # We get all the installed files that were not packaged by finding entries in the installed file that are - # not in the packaged file. - comm -23 /tmp/installed-files /tmp/packaged-files > /tmp/unpackaged-files - # If there are no unpackaged files something else went wrong. - if [ ! -s /tmp/unpackaged-files ]; then - exit 1 - fi - - # Otherwise, we append the unpackaged files to the filelist for the systemd package and retry the build. - cat /tmp/unpackaged-files >>debian/systemd.install - build -fi - -( - shopt -s nullglob - rm -f "$BUILDDIR"/*.deb "$BUILDDIR"/*.ddeb - - cp ../*.deb ../*.ddeb "$PACKAGEDIR" - cp ../*.deb ../*.ddeb "$OUTPUTDIR" - cp ../*.deb ../*.ddeb "$BUILDDIR" - # These conflict with the packages that we actually want to install, so remove them - rm -f "$BUILDDIR"/systemd-standalone-*.deb "$BUILDDIR"/systemd-standalone-*.ddeb -) diff --git a/mkosi.conf.d/10-debian-ubuntu/mkosi.conf b/mkosi.conf.d/10-debian-ubuntu/mkosi.conf index 07bf95cee7..41f1972e1f 100644 --- a/mkosi.conf.d/10-debian-ubuntu/mkosi.conf +++ b/mkosi.conf.d/10-debian-ubuntu/mkosi.conf @@ -8,12 +8,6 @@ Distribution=|ubuntu PackageManagerTrees=mkosi-pinning.pref:/etc/apt/preferences.d/mkosi-pinning.pref [Content] -Environment= - GIT_URL=https://salsa.debian.org/systemd-team/systemd.git - GIT_SUBDIR=debian - GIT_BRANCH=debian/master - GIT_COMMIT=abf24e775c67cf054f474526dd5d9d952a00228b - VolatilePackages= libnss-myhostname libnss-mymachines @@ -21,6 +15,8 @@ VolatilePackages= libnss-systemd libpam-systemd libsystemd-dev + libsystemd-shared + libsystemd0 libudev-dev systemd systemd-container @@ -40,17 +36,12 @@ VolatilePackages= udev Packages= - ^libasan[0-9]+$ - ^libtss2-esys-[0-9.]+-0$ - ^libtss2-mu-[0-9.]+-0$ - ^libubsan[0-9]+$ apt bind9-dnsutils cryptsetup-bin dbus-broker dbus-user-session dmsetup - dpkg-dev f2fs-tools fdisk git-core @@ -59,9 +50,6 @@ Packages= iputils-ping isc-dhcp-server libcap-ng-utils - libclang-rt-dev - libtss2-rc0 - libtss2-tcti-device0 locales man-db multipath-tools @@ -85,7 +73,6 @@ Packages= xxd InitrdPackages= - libclang-rt-dev tpm2-tools InitrdVolatilePackages= diff --git a/mkosi.conf.d/10-debian-ubuntu/mkosi.postinst b/mkosi.conf.d/10-debian-ubuntu/mkosi.postinst deleted file mode 100755 index 314f235f5f..0000000000 --- a/mkosi.conf.d/10-debian-ubuntu/mkosi.postinst +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: LGPL-2.1-or-later -set -e - -# By default Suggests are not installed (and often Recommends are disabled too), which means we will miss -# the dlopen optional dependencies, but the tests need them, so parse them from the package metadata and -# install them. This is not an issue when building locally, as the build and runtime images are the same, -# so they would get installed as build dependencies anyway. - -if [ "$1" = "build" ] || ! ((NO_BUILD)); then - exit 0 -fi - -# Query the Recommends and Suggests of all systemd packages, by matching on the version -systemd_version="$(dpkg-query --showformat '${Version}' --show systemd)" -mapfile -t systemd_packages < <( dpkg --list | grep '^ii' | grep "$systemd_version" | awk '{print $2}' | tr '\n' ' ' ) -extra_packages=() -# shellcheck disable=SC2068 -for package in ${systemd_packages[@]}; do - # We are looking for dlopens, so filter for libraries - mapfile -t -O "${#extra_packages[@]}" extra_packages < <(dpkg-query --showformat '${Suggests}' --show "$package" | sed -e "s/, /\n/g" -e "s/|.*//" | grep "lib") - mapfile -t -O "${#extra_packages[@]}" extra_packages < <(dpkg-query --showformat '${Recommends}' --show "$package" | sed -e "s/, /\n/g" -e "s/|.*//" | grep "lib") -done - -if [ "${#extra_packages[@]}" -eq 0 ]; then - exit 0 -fi - -apt install "${extra_packages[@]}" diff --git a/mkosi.conf.d/10-debian-ubuntu/mkosi.prepare b/mkosi.conf.d/10-debian-ubuntu/mkosi.prepare index 645671a031..acab113b8c 100755 --- a/mkosi.conf.d/10-debian-ubuntu/mkosi.prepare +++ b/mkosi.conf.d/10-debian-ubuntu/mkosi.prepare @@ -2,17 +2,15 @@ # SPDX-License-Identifier: LGPL-2.1-or-later set -e -if [ "$1" = "build" ] || ((NO_BUILD)); then +if [[ "$1" == "build" ]]; then exit 0 fi -# shellcheck source=/dev/null -. "$BUILDROOT/usr/lib/os-release" +mapfile -t PACKAGES < <(jq --raw-output .VolatilePackages[] <"$MKOSI_CONFIG") -if [ ! -d "pkg/$ID/debian" ]; then - echo "deb rules not found at pkg/$ID/debian, run mkosi once with -ff to make sure the rules are cloned" >&2 - exit 1 -fi - -cd "pkg/$ID" -DEB_BUILD_PROFILES="pkg.systemd.upstream" apt-get build-dep . +apt-cache depends "${PACKAGES[@]}" | + grep --invert-match --regexp "<" --regexp "|" --regexp systemd | # Remove e.g. and |dbus-broker like results + grep --extended-regexp "Depends|Suggests|Recommends" | + sed --quiet 's/.*: //p' | # Get every line with ": " in it and strip it at the same time. + sort --unique | + xargs --delimiter '\n' --no-run-if-empty mkosi-install diff --git a/mkosi.conf.d/10-fedora/mkosi.conf b/mkosi.conf.d/10-fedora/mkosi.conf index ff716e27df..1b76d9ac94 100644 --- a/mkosi.conf.d/10-fedora/mkosi.conf +++ b/mkosi.conf.d/10-fedora/mkosi.conf @@ -7,11 +7,6 @@ Distribution=fedora Release=rawhide [Content] -Environment= - GIT_URL=https://src.fedoraproject.org/rpms/systemd.git - GIT_BRANCH=rawhide - GIT_COMMIT=a3524fc837f5e7b68f86b3e0a9d470a94a04c4c8 - Packages= compsize dnf5 diff --git a/mkosi.conf.d/10-opensuse/mkosi.build.chroot b/mkosi.conf.d/10-opensuse/mkosi.build.chroot deleted file mode 100755 index 0c598eae56..0000000000 --- a/mkosi.conf.d/10-opensuse/mkosi.build.chroot +++ /dev/null @@ -1,145 +0,0 @@ -#!/bin/bash -# SPDX-License-Identifier: LGPL-2.1-or-later -set -e - -. mkosi.functions - -if ((NO_BUILD)); then - exit 0 -fi - -# shellcheck source=/dev/null -. /usr/lib/os-release -ID="${ID%-*}" - -if [ ! -f "pkg/$ID/systemd.spec" ]; then - echo "spec not found at pkg/$ID/systemd.spec, run mkosi once with -ff to make sure the spec is cloned" >&2 - exit 1 -fi - -if [ -d .git/ ] && [ -z "$(git status --porcelain)" ]; then - TS="$(git show --no-patch --format=%ct HEAD)" -else - TS="${SOURCE_DATE_EPOCH:-$(date +%s)}" -fi - -# The openSUSE filelists hardcode the manpage compression extension. This causes rpmbuild errors since we -# disable manpage compression as the files cannot be found. Fix the issue by removing the compression -# extension. -find "pkg/$ID" -name "files.*" -exec sed --in-place 's/\.gz$//' {} \; - -if systemd-analyze compare-versions "$(rpm --version | cut -d ' ' -f3)" lt "4.20"; then - # Fix the %install override so debuginfo packages are generated. - tee --append /usr/lib/rpm/suse/macros <<'EOF' -%install %{debug_package}\ -%%install\ -%{nil} -EOF -fi - -VERSION="$(cat meson.version)" -RELEASE="$(date "+%Y%m%d%H%M%S" --date "@$TS")" - -DIST="$(rpm --eval %dist)" -ARCH="$(rpm --eval %_arch)" -SRCDEST="/usr/src/debug/systemd-$VERSION-${RELEASE}${DIST}.$ARCH" - -MKOSI_CFLAGS="-O0 -Wp,-U_FORTIFY_SOURCE" -if ((WITH_DEBUG)); then - MKOSI_CFLAGS="$MKOSI_CFLAGS -fdebug-prefix-map=../src=$SRCDEST" -fi -if ((LLVM)); then - # TODO: Remove -fno-sanitize-function when https://github.com/systemd/systemd/issues/29972 is fixed. - MKOSI_CFLAGS="$MKOSI_CFLAGS -shared-libasan -fno-sanitize=function" -fi - -MKOSI_LDFLAGS="$(rpm --eval "%{?build_ldflags}")" -if ((LLVM)) && [[ -n "$SANITIZERS" ]]; then - MKOSI_LDFLAGS="$MKOSI_LDFLAGS -Wl,-rpath=$(clang --print-file-name="")lib/linux" -fi - -# A macro can't have an empty body and currently opensuse does not specify any of its own linker flags so -# set LDFLAGS to %{nil} if there are no linker flags. -if [[ -z "${MKOSI_LDFLAGS// }" ]]; then - MKOSI_LDFLAGS="%{nil}" -fi - -MKOSI_MESON_OPTIONS="-D mode=developer -D b_sanitize=${SANITIZERS:-none}" -if ((WIPE)) && [[ -d "$BUILDDIR/meson-private" ]]; then - MKOSI_MESON_OPTIONS="$MKOSI_MESON_OPTIONS --wipe" -fi - -# TODO: Drop when the spec is fixed (either the patch is adapted or not applied when building for upstream). -sed --in-place '/0009-pid1-handle-console-specificities-weirdness-for-s390.patch/d' "pkg/$ID/systemd.spec" - -build() { - IFS= - # shellcheck disable=SC2046 - env \ - --unset CFLAGS \ - --unset CXXFLAGS \ - --unset LDFLAGS \ - CC="$( ((LLVM)) && echo clang || echo gcc)" \ - CXX="$( ((LLVM)) && echo clang++ || echo g++)" \ - CC_LD="$( ((LLVM)) && echo lld)" \ - CXX_LD="$( ((LLVM)) && echo lld)" \ - rpmbuild \ - -bb \ - --build-in-place \ - --with upstream \ - $( ((WITH_TESTS)) || echo "--nocheck") \ - --define "_topdir /var/tmp" \ - --define "_sourcedir pkg/$ID" \ - --define "_rpmdir $OUTPUTDIR" \ - ${BUILDDIR:+"--define=_vpath_builddir $BUILDDIR"} \ - --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" \ - --define "_binary_payload w.ufdio" \ - $( ((WITH_DEBUG)) || echo "--define=debug_package %{nil}") \ - --define "vendor openSUSE" \ - --define "version_override $VERSION" \ - --define "release_override $RELEASE" \ - --define "__check_files sh -c '$(rpm --define "_topdir /var/tmp" --eval %__check_files) | tee /tmp/unpackaged-files'" \ - --define "build_cflags $(rpm --eval "%{?build_cflags}") $MKOSI_CFLAGS $CFLAGS" \ - --define "build_cxxflags $(rpm --eval "%{?build_cxxflags}") $MKOSI_CFLAGS $CFLAGS" \ - --define "build_ldflags $MKOSI_LDFLAGS $LDFLAGS" \ - $( ((MESON_VERBOSE)) || echo "--undefine=__meson_verbose") \ - --define "meson_extra_configure_options $MKOSI_MESON_OPTIONS $MESON_OPTIONS" \ - --define "__os_install_post /usr/lib/rpm/brp-suse %{nil}" \ - --define "__elf_exclude_path ^/usr/lib/systemd/tests/unit-tests/.*$" \ - --define "__script_requires %{nil}" \ - --define "_find_debuginfo_dwz_opts %{nil}" \ - --define "_fixperms true" \ - --noclean \ - "$@" \ - "pkg/$ID/systemd.spec" - - EXIT_STATUS=$? - - # Make sure we don't reconfigure twice. - MKOSI_MESON_OPTIONS="${MKOSI_MESON_OPTIONS//"--wipe"/}" - - return $EXIT_STATUS -} - -if ! build; then - if [ ! -s /tmp/unpackaged-files ]; then - exit 1 - fi - - # rpm will append to any existing systemd.lang so delete it explicitly so we don't get duplicate file - # warnings. - rm systemd.lang - - grep -v ".debug" /tmp/unpackaged-files >>"pkg/$ID/files.systemd" - build --noprep --nocheck -fi - -( - shopt -s nullglob - rm -f "$BUILDDIR"/*.rpm -) - -cp "$OUTPUTDIR"/*.rpm "$PACKAGEDIR" -cp "$OUTPUTDIR"/*.rpm "$BUILDDIR" - -make_sysext_unsigned /var/tmp/BUILD/*/BUILDROOT diff --git a/mkosi.conf.d/10-opensuse/mkosi.conf b/mkosi.conf.d/10-opensuse/mkosi.conf index 06edcbae41..df01c3b6be 100644 --- a/mkosi.conf.d/10-opensuse/mkosi.conf +++ b/mkosi.conf.d/10-opensuse/mkosi.conf @@ -12,12 +12,9 @@ Repositories=non-oss PackageManagerTrees=macros.db_backend:/etc/rpm/macros.db_backend [Content] -Environment= - GIT_URL=https://code.opensuse.org/package/systemd - GIT_BRANCH=master - GIT_COMMIT=6812406e52a474568744c267e7bade1496bb26a5 - VolatilePackages= + libsystemd0 + libudev1 systemd systemd-boot systemd-container @@ -43,23 +40,14 @@ Packages= docbook-xsl-stylesheets f2fs-tools gawk - gcc-c++ git-core glibc-locale-base gnutls grep - group(bin) - group(daemon) - group(games) - group(nobody) - group(root) gzip iputils kernel-default kmod - libasan8 - libkmod2 - libubsan1 multipath-tools ncat open-iscsi @@ -74,7 +62,6 @@ Packages= python3-pexpect python3-psutil quota - rpm-build rsync sbsigntools sed @@ -85,20 +72,13 @@ Packages= tgt timezone tpm2.0-tools - user(bin) - user(daemon) - user(games) - user(nobody) - user(root) veritysetup vim xz zypper InitrdPackages= - clang kmod - libkmod2 tpm2.0-tools InitrdVolatilePackages= diff --git a/mkosi.conf.d/10-opensuse/mkosi.prepare b/mkosi.conf.d/10-opensuse/mkosi.prepare index c57aa878b8..6ee0af2de9 100755 --- a/mkosi.conf.d/10-opensuse/mkosi.prepare +++ b/mkosi.conf.d/10-opensuse/mkosi.prepare @@ -2,63 +2,22 @@ # SPDX-License-Identifier: LGPL-2.1-or-later set -e -if [ "$1" = "build" ] || ((NO_BUILD)); then +if [[ "$1" == "build" ]]; then exit 0 fi -# shellcheck source=/dev/null -. "$BUILDROOT/usr/lib/os-release" -ID="${ID%-*}" - -if [ ! -f "pkg/$ID/systemd.spec" ]; then - echo "spec not found at pkg/$ID/systemd.spec, run mkosi once with -ff to make sure the spec is cloned" >&2 - exit 1 -fi - -# TODO: Drop when the spec is fixed (either the patch is adapted or not applied when building for upstream). -sed --in-place '/0009-pid1-handle-console-specificities-weirdness-for-s390.patch/d' "pkg/$ID/systemd.spec" - -for DEPS in --requires --buildrequires; do - mkosi-chroot \ - rpmspec \ - --with upstream \ - --query \ - "$DEPS" \ - --define "_topdir /var/tmp" \ - --define "_sourcedir pkg/$ID" \ - "pkg/$ID/systemd.spec" | - grep --invert-match --regexp systemd --regexp /bin/sh --regexp "rpmlib(" --regexp udev | - sort --unique | - tee /tmp/buildrequires | - xargs --delimiter '\n' mkosi-install -done - -until mkosi-chroot \ - rpmbuild \ - -bd \ - --build-in-place \ - --with upstream \ - --define "_topdir /var/tmp" \ - --define "_sourcedir pkg/$ID" \ - --define "_build_name_fmt %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm" \ - "pkg/$ID/systemd.spec" -do - EXIT_STATUS=$? - if [ $EXIT_STATUS -ne 11 ]; then - exit $EXIT_STATUS - fi - - mkosi-chroot \ - rpm \ - --query \ - --package \ - --requires \ - /var/tmp/SRPMS/systemd-*.buildreqs.nosrc.rpm | - grep --invert-match '^rpmlib(' | - sort --unique >/tmp/dynamic-buildrequires - - sort /tmp/buildrequires /tmp/dynamic-buildrequires | - uniq --unique | - tee --append /tmp/buildrequires | - xargs --delimiter '\n' mkosi-install -done +DEPS="" + +while read -r PACKAGE; do + # zypper's output is not machine readable so we make do with sed instead. + DEPS="$DEPS\n$( + zypper info --requires --recommends --suggests "$PACKAGE" | + sed '/Requires/,$!d' | # Remove everything before Requires line + sed --quiet 's/^ //p' # All indented lines have dependencies + )" +done < <(jq --raw-output .VolatilePackages[] <"$MKOSI_CONFIG") + +echo -e "$DEPS" | + grep --invert-match --regexp systemd --regexp udev --regexp qemu | + sort --unique | + xargs --delimiter '\n' --no-run-if-empty mkosi-install diff --git a/mkosi.conf.d/20-build.conf b/mkosi.conf.d/20-build.conf new file mode 100644 index 0000000000..8c16d9b9f8 --- /dev/null +++ b/mkosi.conf.d/20-build.conf @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later + +# Add a dependency on the build image unless NO_BUILD=1. + +[Match] +Environment=!NO_BUILD=1 + +[Config] +Dependencies=build diff --git a/mkosi.conf.d/20-none.conf b/mkosi.conf.d/20-none.conf index 57a2f3f0e3..0e4d919a27 100644 --- a/mkosi.conf.d/20-none.conf +++ b/mkosi.conf.d/20-none.conf @@ -1,9 +1,11 @@ # SPDX-License-Identifier: LGPL-2.1-or-later -# If we're only rerunning the build script, remove all subimage dependencies to speed up builds. +# If we're only rerunning the build script, remove all subimage dependencies except the build image to speed +# up builds. [Match] Format=none [Config] Dependencies= +Dependencies=build diff --git a/mkosi.conf.d/20-sanitizers.conf b/mkosi.conf.d/20-sanitizers.conf index 235b233e1a..62d052360a 100644 --- a/mkosi.conf.d/20-sanitizers.conf +++ b/mkosi.conf.d/20-sanitizers.conf @@ -2,6 +2,7 @@ [Match] Environment=SANITIZERS +Environment=!SANITIZERS= [Content] # Set verify_asan_link_order=0 to prevent ASAN warnings when building the image and make sure the real ASAN @@ -17,3 +18,6 @@ KernelCommandLine= systemd.setenv=UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1 LSAN_OPTIONS=suppressions=/usr/lib/systemd/leak-sanitizer-suppressions systemd.setenv=LSAN_OPTIONS=suppressions=/usr/lib/systemd/leak-sanitizer-suppressions + +[Config] +Include=%D/mkosi.sanitizers -- cgit v1.2.3