mkosi: Disable BuildSourcesEphemeral=

Let's stop using BuildSourcesEphemeral= and instead make sure we don't
generate any auxiliary files during the mkosi build process.

We achieve this through a combination of trap to remove any new files
we create and bind mounts from /tmp over existing files whenever we need
to modify an existing file.

We also add a CI step to ensure we don't regress

.github/workflows/mkosi.yml

@@ -137,6 +137,7 @@ jobs:
           mkfs.btrfs btrfs.raw
           sudo mkdir /mnt/mkosi
           LOOP="$(sudo losetup --find --show --direct-io=on btrfs.raw)"
+          rm btrfs.raw
           sudo mount "$LOOP" /mnt/mkosi --options compress=zstd:1,user_subvol_rm_allowed,noatime,discard=async,space_cache=v2
           sudo chown "$(id -u):$(id -g)" /mnt/mkosi
           mkdir /mnt/mkosi/tmp
@@ -145,9 +146,6 @@ jobs:
 
       - name: Configure
         run: |
-          # XXX: drop after the HyperV bug that breaks secure boot KVM guests is solved
-          sed -i "s/'firmware'\s*:\s*'auto'/'firmware' : 'uefi'/g" test/*/meson.build
-
           tee mkosi.local.conf <<EOF
           [Distribution]
           Distribution=${{ matrix.distro }}
@@ -202,6 +200,17 @@ jobs:
       - name: Build image
         run: sudo mkosi sandbox -- meson compile -C build mkosi
 
+      - name: Make sure sources weren't polluted by package build scripts
+        run: |
+          git status
+          git diff
+          test -z "$(git status --porcelain)"
+          # There will only be one subdirectory, so we don't have to know exactly which one. Also, the
+          # subdirectory will be owned by root since we used sudo to build the image, so use sudo here as well.
+          sudo git -C pkg/* status
+          sudo git -C pkg/* diff
+          test -z "$(sudo git -C pkg/* status --porcelain)"
+
       - name: Run integration tests
         run: |
           if [[ "$(sudo mkosi sandbox -- meson test --help)" == *"--max-lines"* ]]; then

mkosi.clangd

@@ -14,7 +14,6 @@ BUILDSUBDIR="$(jq -r .BuildSubdirectory <<< "$MKOSI_CONFIG")"
 
 exec "${SPAWN[@]}" mkosi \
     --incremental=strict \
-    --build-sources-ephemeral=no \
     --format=none \
     build \
     -- \

mkosi.conf.d/20-build.conf

@@ -7,6 +7,3 @@ Environment=!NO_BUILD=1
 
 [Config]
 Dependencies=build
-
-[Build]
-BuildSourcesEphemeral=yes

mkosi.images/build/mkosi.conf.d/arch/mkosi.build.chroot

@@ -7,16 +7,6 @@ if [[ ! -f "pkg/$PKG_SUBDIR/PKGBUILD" ]]; then
     exit 1
 fi
 
-# We can't configure the source or build directory so we use symlinks instead to make sure they are in the
-# expected locations. Because we run with --noextract we are responsible for making sure the source files
-# appear in src/. This means not only the systemd source directory, but also the patches and configuration
-# files that are shipped in the packaging repository. To achieve this, instead of symlinking the systemd
-# sources and build directory directly into "pkg/$PKG_SUBDIR/src", we symlink them into "pkg/$PKG_SUBDIR" and
-# then symlink "pkg/$PKG_SUBDIR" to "pkg/$PKG_SUBDIR/src".
-ln --symbolic "$SRCDIR" "pkg/$PKG_SUBDIR/systemd"
-ln --symbolic "$BUILDDIR" "pkg/$PKG_SUBDIR/build"
-ln --symbolic . "pkg/$PKG_SUBDIR/src"
-
 MKOSI_CFLAGS="-O0 -g -Wp,-U_FORTIFY_SOURCE"
 if ((LLVM)); then
     # TODO: Remove -fno-sanitize-function when https://github.com/systemd/systemd/issues/29972 is fixed.
@@ -64,15 +54,27 @@ EOF
 
 TS="${SOURCE_DATE_EPOCH:-$(date +%s)}"
 
-sed --in-place "pkg/$PKG_SUBDIR/PKGBUILD" \
+sed "pkg/$PKG_SUBDIR/PKGBUILD" \
     --expression "s/^_tag=.*/_tag=$(cat meson.version)/" \
-    --expression "s/^pkgrel=.*/pkgrel=$(date "+%Y%m%d%H%M%S" --date "@$TS")/"
+    --expression "s/^pkgrel=.*/pkgrel=$(date "+%Y%m%d%H%M%S" --date "@$TS")/" >/tmp/PKGBUILD
+mount --bind /tmp/PKGBUILD "pkg/$PKG_SUBDIR/PKGBUILD"
 
 (
     shopt -s nullglob
     rm -f "$BUILDDIR"/*.pkg.tar
 )
 
+# We can't configure the source or build directory so we use symlinks instead to make sure they are in the
+# expected locations. Because we run with --noextract we are responsible for making sure the source files
+# appear in src/. This means not only the systemd source directory, but also the patches and configuration
+# files that are shipped in the packaging repository. To achieve this, instead of symlinking the systemd
+# sources and build directory directly into "pkg/$PKG_SUBDIR/src", we symlink them into "pkg/$PKG_SUBDIR" and
+# then symlink "pkg/$PKG_SUBDIR" to "pkg/$PKG_SUBDIR/src".
+trap 'rm -f "pkg/$PKG_SUBDIR/src" "pkg/$PKG_SUBDIR/build" "pkg/$PKG_SUBDIR/systemd"' EXIT
+ln --symbolic "$SRCDIR" "pkg/$PKG_SUBDIR/systemd"
+ln --symbolic "$BUILDDIR" "pkg/$PKG_SUBDIR/build"
+ln --symbolic . "pkg/$PKG_SUBDIR/src"
+
 # We get around makepkg's root check by setting EUID to something else.
 # Linting the PKGBUILD takes multiple seconds every build so avoid that by nuking all the linting functions.
 # shellcheck disable=SC2046
@@ -83,6 +85,7 @@ env --chdir="pkg/$PKG_SUBDIR" \
     --noextract \
     --noprepare \
     $( ((WITH_TESTS)) || echo --nocheck) \
+    --clean \
     --force \
     _systemd_UPSTREAM=1 \
     BUILDDIR="$PWD/pkg/$PKG_SUBDIR" \

mkosi.images/build/mkosi.conf.d/centos-fedora/mkosi.build.chroot

@@ -107,7 +107,6 @@ CXX_LD="$( ((LLVM)) && echo lld)" \
     --define "_find_debuginfo_opts --unique-debug-src-base \"%{name}\"" \
     --define "_find_debuginfo_dwz_opts %{nil}" \
     --undefine _package_note_flags \
-    --noclean \
     "pkg/$PKG_SUBDIR/systemd.spec"
 
 cp "$BUILDDIR"/*.rpm "$PACKAGEDIR"

mkosi.images/build/mkosi.conf.d/centos-fedora/mkosi.prepare

@@ -27,7 +27,8 @@ mkosi-chroot \
 # rpmbuild -br tries to build a source package which means all source files have to exist which isn't the
 # case when using --build-in-place so we get rid of the source file that doesn't exist to make it happy.
 # TODO: Use -bd instead of -br and get rid of this once we don't need to build on CentOS Stream 9 anymore.
-sed '/Source0/d' --in-place "pkg/$PKG_SUBDIR/systemd.spec"
+sed '/Source0/d' "pkg/$PKG_SUBDIR/systemd.spec" >/tmp/systemd.spec
+mount --bind /tmp/systemd.spec "pkg/$PKG_SUBDIR/systemd.spec"
 
 until mkosi-chroot \
     rpmbuild \

mkosi.images/build/mkosi.conf.d/debian-ubuntu/mkosi.build.chroot

@@ -7,32 +7,6 @@ if [[ ! -d "pkg/$PKG_SUBDIR/debian" ]]; then
     exit 1
 fi
 
-# We transplant the debian/ folder from the deb package sources into the upstream sources.
-mount --mkdir --bind "$SRCDIR/pkg/$PKG_SUBDIR/debian" "$SRCDIR"/debian
-
-# We remove the patches so they don't get applied.
-rm -rf "$SRCDIR"/debian/patches/*
-
-# While the build directory can be specified through DH_OPTIONS, the default one is hardcoded everywhere so
-# we have to use that. Because it is architecture dependent, we query it using dpkg-architecture first.
-DEB_HOST_GNU_TYPE="$(dpkg-architecture --query DEB_HOST_GNU_TYPE)"
-mount --mkdir --bind "$BUILDDIR" "$SRCDIR/obj-$DEB_HOST_GNU_TYPE"
-
-TS="${SOURCE_DATE_EPOCH:-$(date +%s)}"
-
-# Add a new changelog entry to update the version. We use a fixed date since a dynamic one causes a full
-# rebuild every time.
-cat >debian/changelog.new <<EOF
-systemd ($(cat meson.version)-$(date "+%Y%m%d%H%M%S" --date "@$TS")) UNRELEASED; urgency=low
-
-  * Automatic build from mkosi
-
- -- systemd test <systemd-devel@lists.freedesktop.org>  $(date --rfc-email --date "@$TS")
-
-EOF
-cat debian/changelog >>debian/changelog.new
-mv debian/changelog.new debian/changelog
-
 MKOSI_CFLAGS="-O0 -g"
 if ((LLVM)); then
     # TODO: Remove -fno-sanitize-function when https://github.com/systemd/systemd/issues/29972 is fixed.
@@ -53,6 +27,32 @@ if ((COVERAGE)); then
     MKOSI_CFLAGS="$MKOSI_CFLAGS -fprofile-dir=/coverage"
 fi
 
+# We transplant the debian/ folder from the deb package sources into the upstream sources.
+mount --mkdir --bind "$SRCDIR/pkg/$PKG_SUBDIR/debian" "$SRCDIR"/debian
+
+# We remove the patches so they don't get applied.
+rm -rf "$SRCDIR"/debian/patches/*
+
+# While the build directory can be specified through DH_OPTIONS, the default one is hardcoded everywhere so
+# we have to use that. Because it is architecture dependent, we query it using dpkg-architecture first.
+DEB_HOST_GNU_TYPE="$(dpkg-architecture --query DEB_HOST_GNU_TYPE)"
+mount --mkdir --bind "$BUILDDIR" "$SRCDIR/obj-$DEB_HOST_GNU_TYPE"
+
+TS="${SOURCE_DATE_EPOCH:-$(date +%s)}"
+
+# Add a new changelog entry to update the version. We use a fixed date since a dynamic one causes a full
+# rebuild every time.
+cat >/tmp/changelog <<EOF
+systemd ($(cat meson.version)-$(date "+%Y%m%d%H%M%S" --date "@$TS")) UNRELEASED; urgency=low
+
+  * Automatic build from mkosi
+
+ -- systemd test <systemd-devel@lists.freedesktop.org>  $(date --rfc-email --date "@$TS")
+
+EOF
+cat debian/changelog >>/tmp/changelog
+mount --bind /tmp/changelog "$PWD/debian/changelog"
+
 # TODO: Drop GENSYMBOLS_LEVEL once https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=986746 is fixed.
 build() {
     env \
@@ -94,6 +94,8 @@ build() {
     return $EXIT_STATUS
 }
 
+trap 'umount "$SRCDIR/obj-$DEB_HOST_GNU_TYPE" && dpkg-buildpackage -T clean' EXIT
+
 if ! build; then
     # debhelper installs files for each package to debian/<package> so we figure out which files were
     # packaged by querying all the package names from debian/control and running find on each of the
@@ -131,7 +133,9 @@ if ! build; then
     echo -e "::warning title=UNPACKAGED-FILES::The following files were not picked up by the package, please update the packaging recipe to include: $(tr '\n' ' ' </tmp/unpackaged-files)"
 
     # Otherwise, we append the unpackaged files to the filelist for the systemd package and retry the build.
-    cat /tmp/unpackaged-files >>debian/systemd.install
+    cp debian/systemd.install /tmp/systemd.install
+    cat /tmp/unpackaged-files >>/tmp/systemd.install
+    mount --bind /tmp/systemd.install "$PWD/debian/systemd.install"
     build
 fi
 

mkosi.images/build/mkosi.conf.d/opensuse/mkosi.build.chroot

@@ -14,7 +14,10 @@ TS="${SOURCE_DATE_EPOCH:-$(date +%s)}"
 # The openSUSE filelists hardcode the manpage compression extension. This causes rpmbuild errors since we
 # disable manpage compression as the files cannot be found. Fix the issue by removing the compression
 # extension.
-find "pkg/$PKG_SUBDIR${GIT_SUBDIR:+/$GIT_SUBDIR}" -name "files.*" -exec sed --in-place 's/\.gz$//' {} \;
+while read -r filelist; do
+    sed 's/\.gz$//' "$filelist" >"/tmp/$(basename "$filelist")"
+    mount --bind "/tmp/$(basename "$filelist")" "$filelist"
+done < <(find "pkg/$PKG_SUBDIR${GIT_SUBDIR:+/$GIT_SUBDIR}" -name "files.*")
 
 if [[ "$(rpm --eval "%{lua:print(rpm.vercmp('$(rpm --version | cut -d ' ' -f3)', '4.20'))}")" == "-1" ]]; then
     # Fix the %install override so debuginfo packages are generated.
@@ -95,12 +98,20 @@ build() {
         --define "__script_requires %{nil}" \
         --define "_find_debuginfo_dwz_opts %{nil}" \
         --define "_find_debuginfo_opts --unique-debug-src-base \"%{name}\"" \
-        --noclean \
         "$@" \
         "pkg/$PKG_SUBDIR${GIT_SUBDIR:+/$GIT_SUBDIR}/systemd.spec"
 
     EXIT_STATUS=$?
 
+    # TODO: Remove when the opensuse spec cleans these up itself (https://build.opensuse.org/request/show/1251175).
+    rm -f \
+        systemd.lang \
+        debugfiles.list \
+        debuglinks.list \
+        debugsourcefiles.list \
+        debugsources.list \
+        elfbins.list
+
     # Make sure we don't reconfigure twice.
     MKOSI_MESON_OPTIONS="${MKOSI_MESON_OPTIONS//"--wipe"/}"
 
@@ -117,10 +128,6 @@ if ! build; then
         exit 1
     fi
 
-    # rpm will append to any existing systemd.lang so delete it explicitly so we don't get duplicate file
-    # warnings.
-    rm systemd.lang
-
     grep -v ".debug" /tmp/unpackaged-files >>"pkg/$PKG_SUBDIR${GIT_SUBDIR:+/$GIT_SUBDIR}/files.systemd"
     build --noprep --nocheck
 fi

test/integration-test-wrapper.py

@@ -508,7 +508,11 @@ def main() -> None:
         '--runtime-scratch=no',
         *([f'--qemu-args=-rtc base={rtc}'] if rtc else []),
         *args.mkosi_args,
-        '--firmware', args.firmware,
+        # mkosi will use the UEFI secure boot firmware by default on UEFI platforms. However, this breaks on
+        # Github Actions in combination with KVM because of a HyperV bug so make sure we use the non secure
+        # boot firmware on Github Actions.
+        # TODO: Drop after the HyperV bug that breaks secure boot KVM guests is solved
+        '--firmware', 'uefi' if args.firmware == 'auto' and os.getenv("GITHUB_ACTIONS") else args.firmware,
         *(['--kvm', 'no'] if int(os.getenv('TEST_NO_KVM', '0')) else []),
         '--kernel-command-line-extra',
         ' '.join(