Merge pull request #20910 from poettering/nftw-no-more

basic: add recurse_dir() function as modern replacement for nftw()

18 files changed, 914 insertions, 146 deletions

diff --git a/TODO b/TODO
index e54d5447fd..d8975158b9 100644
--- a/TODO
+++ b/TODO
@@ -7,8 +7,6 @@ Bugfixes:
 * userdbctl: "Password OK: yes" is shown even when there are no passwords
   or the password is locked.
 
-* Get rid of nftw(). We should refuse to use such useless APIs on principle.
-
 * Jun 01 09:43:02 krowka systemd[1]: Unit user@1000.service has alias user@.service.
   Jun 01 09:43:02 krowka systemd[1]: Unit user@6.service has alias user@.service.
   Jun 01 09:43:02 krowka systemd[1]: Unit user-runtime-dir@6.service has alias user-runtime-dir@.service.
diff --git a/src/basic/cgroup-util.c b/src/basic/cgroup-util.c
index e60db5e48c..37a5a530f3 100644
--- a/src/basic/cgroup-util.c
+++ b/src/basic/cgroup-util.c
@@ -1,7 +1,6 @@
 /* SPDX-License-Identifier: LGPL-2.1-or-later */
 
 #include <errno.h>
-#include <ftw.h>
 #include <limits.h>
 #include <signal.h>
 #include <stddef.h>
diff --git a/src/basic/dirent-util.c b/src/basic/dirent-util.c
index aba14481df..a70871b33d 100644
--- a/src/basic/dirent-util.c
+++ b/src/basic/dirent-util.c
@@ -7,6 +7,18 @@
 #include "path-util.h"
 #include "string-util.h"
 
+int stat_mode_to_dirent_type(mode_t mode) {
+        return
+                S_ISREG(mode)  ? DT_REG  :
+                S_ISDIR(mode)  ? DT_DIR  :
+                S_ISLNK(mode)  ? DT_LNK  :
+                S_ISFIFO(mode) ? DT_FIFO :
+                S_ISSOCK(mode) ? DT_SOCK :
+                S_ISCHR(mode)  ? DT_CHR  :
+                S_ISBLK(mode)  ? DT_BLK  :
+                                 DT_UNKNOWN;
+}
+
 static int dirent_ensure_type(DIR *d, struct dirent *de) {
         struct stat st;
 
@@ -24,15 +36,7 @@ static int dirent_ensure_type(DIR *d, struct dirent *de) {
         if (fstatat(dirfd(d), de->d_name, &st, AT_SYMLINK_NOFOLLOW) < 0)
                 return -errno;
 
-        de->d_type =
-                S_ISREG(st.st_mode)  ? DT_REG  :
-                S_ISDIR(st.st_mode)  ? DT_DIR  :
-                S_ISLNK(st.st_mode)  ? DT_LNK  :
-                S_ISFIFO(st.st_mode) ? DT_FIFO :
-                S_ISSOCK(st.st_mode) ? DT_SOCK :
-                S_ISCHR(st.st_mode)  ? DT_CHR  :
-                S_ISBLK(st.st_mode)  ? DT_BLK  :
-                                       DT_UNKNOWN;
+        de->d_type = stat_mode_to_dirent_type(st.st_mode);
 
         return 0;
 }
diff --git a/src/basic/dirent-util.h b/src/basic/dirent-util.h
index c7956e7c1b..5bbd54df5a 100644
--- a/src/basic/dirent-util.h
+++ b/src/basic/dirent-util.h
@@ -8,6 +8,8 @@
 #include "macro.h"
 #include "path-util.h"
 
+int stat_mode_to_dirent_type(mode_t mode);
+
 bool dirent_is_file(const struct dirent *de) _pure_;
 bool dirent_is_file_with_suffix(const struct dirent *de, const char *suffix) _pure_;
 
diff --git a/src/basic/fileio.c b/src/basic/fileio.c
index cced1dd564..0a483854f2 100644
--- a/src/basic/fileio.c
+++ b/src/basic/fileio.c
@@ -928,6 +928,9 @@ DIR *xopendirat(int fd, const char *name, int flags) {
 
         assert(!(flags & O_CREAT));
 
+        if (fd == AT_FDCWD && flags == 0)
+                return opendir(name);
+
         nfd = openat(fd, name, O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC|flags, 0);
         if (nfd < 0)
                 return NULL;
diff --git a/src/basic/locale-util.c b/src/basic/locale-util.c
index b6f9a760a4..9a7c7795e7 100644
--- a/src/basic/locale-util.c
+++ b/src/basic/locale-util.c
@@ -2,7 +2,6 @@
 
 #include <errno.h>
 #include <fcntl.h>
-#include <ftw.h>
 #include <langinfo.h>
 #include <libintl.h>
 #include <stddef.h>
diff --git a/src/basic/meson.build b/src/basic/meson.build
index 3bc5196fa0..adb7b666c6 100644
--- a/src/basic/meson.build
+++ b/src/basic/meson.build
@@ -192,6 +192,8 @@ basic_sources = files('''
         raw-reboot.h
         recovery-key.c
         recovery-key.h
+        recurse-dir.c
+        recurse-dir.h
         replace-var.c
         replace-var.h
         rlimit-util.c
diff --git a/src/basic/recurse-dir.c b/src/basic/recurse-dir.c
new file mode 100644
index 0000000000..fe34ffbfd4
--- /dev/null
+++ b/src/basic/recurse-dir.c
@@ -0,0 +1,416 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include "alloc-util.h"
+#include "dirent-util.h"
+#include "fd-util.h"
+#include "fileio.h"
+#include "mountpoint-util.h"
+#include "recurse-dir.h"
+#include "sort-util.h"
+
+#define DEFAULT_RECURSION_MAX 100
+
+static int sort_func(struct dirent * const *a, struct dirent * const *b) {
+        return strcmp((*a)->d_name, (*b)->d_name);
+}
+
+struct dirent** readdir_all_free(struct dirent **array) {
+
+        /* Destructor that relies on the fact that the array of dirent structure pointers is NULL
+         * terminated */
+
+        if (!array)
+                return NULL;
+
+        for (struct dirent **i = array; *i; i++)
+                free(*i);
+
+        return mfree(array);
+}
+
+int readdir_all(DIR *d,
+                RecurseDirFlags flags,
+                struct dirent ***ret) {
+
+        _cleanup_(readdir_all_freep) struct dirent **de_array = NULL;
+        size_t n_de = 0;
+
+        assert(d);
+
+        /* Returns an array with pointers to "struct dirent" directory entries, optionally sorted. Free the
+         * array with readdir_all_freep(). */
+
+        for (;;) {
+                _cleanup_free_ struct dirent *copy = NULL;
+                struct dirent *de;
+
+                errno = 0;
+                de = readdir(d);
+                if (!de) {
+                        if (errno == 0)
+                                break;
+
+                        return -errno;
+                }
+
+                /* Depending on flag either ignore everything starting with ".", or just "." itself and ".." */
+                if (FLAGS_SET(flags, RECURSE_DIR_IGNORE_DOT) ?
+                    de->d_name[0] == '.' :
+                    dot_or_dot_dot(de->d_name))
+                        continue;
+
+                if (n_de >= INT_MAX) /* Make sure we can return the number as 'int' return value */
+                        return -ERANGE;
+
+                if (!GREEDY_REALLOC(de_array, n_de+2))
+                        return -ENOMEM;
+
+                copy = memdup(de, de->d_reclen);
+                if (!copy)
+                        return -ENOMEM;
+
+                de_array[n_de++] = TAKE_PTR(copy);
+                de_array[n_de] = NULL; /* guarantee array remains NUL terminated */
+        }
+
+        if (FLAGS_SET(flags, RECURSE_DIR_SORT))
+                typesafe_qsort(de_array, n_de, sort_func);
+
+        if (ret)
+                *ret = TAKE_PTR(de_array);
+
+        return (int) n_de;
+}
+
+int recurse_dir(
+                DIR *d,
+                const char *path,
+                unsigned statx_mask,
+                unsigned n_depth_max,
+                RecurseDirFlags flags,
+                recurse_dir_func_t func,
+                void *userdata) {
+
+        _cleanup_(readdir_all_freep) struct dirent **de = NULL;
+        int r, n;
+
+        assert(d);
+        assert(func);
+
+        /* This is a lot like ftw()/nftw(), but a lot more modern, i.e. built around openat()/statx(), and
+         * under the assumption that fds are not as 'expensive' as they used to be. */
+
+        if (n_depth_max == 0)
+                return -EOVERFLOW;
+        if (n_depth_max == UINT_MAX) /* special marker for "default" */
+                n_depth_max = DEFAULT_RECURSION_MAX;
+
+        n = readdir_all(d, flags, &de);
+        if (n < 0)
+                return n;
+
+        for (int i = 0; i < n; i++) {
+                _cleanup_free_ char *joined = NULL;
+                _cleanup_closedir_ DIR *subdir = NULL;
+                _cleanup_close_ int inode_fd = -1;
+                STRUCT_STATX_DEFINE(sx);
+                bool sx_valid = false;
+                const char *p;
+
+                /* For each directory entry we'll do one of the following:
+                 *
+                 * 1) If the entry refers to a directory, we'll open it as O_DIRECTORY 'subdir' and then statx() the opened directory if requested
+                 * 2) Otherwise and RECURSE_DIR_INODE_FD is set we'll open O_PATH 'inode_fd' and then statx() the opened inode
+                 * 3) Otherwise we'll statx() the directory entry via the directory we are currently looking at
+                 */
+
+                if (path) {
+                        joined = path_join(path, de[i]->d_name);
+                        if (!joined)
+                                return -ENOMEM;
+
+                        p = joined;
+                } else
+                        p = de[i]->d_name;
+
+                if (IN_SET(de[i]->d_type, DT_UNKNOWN, DT_DIR)) {
+                        subdir = xopendirat(dirfd(d), de[i]->d_name, O_NOFOLLOW);
+                        if (!subdir) {
+                                if (errno == ENOENT) /* Vanished by now, go for next file immediately */
+                                        continue;
+
+                                /* If it is a subdir but we failed to open it, then fail */
+                                if (!IN_SET(errno, ENOTDIR, ELOOP)) {
+                                        log_debug_errno(errno, "Failed to open directory '%s': %m", p);
+
+                                        assert(errno <= RECURSE_DIR_SKIP_OPEN_DIR_ERROR_MAX - RECURSE_DIR_SKIP_OPEN_DIR_ERROR_BASE);
+
+                                        r = func(RECURSE_DIR_SKIP_OPEN_DIR_ERROR_BASE + errno,
+                                                 p,
+                                                 dirfd(d),
+                                                 -1,
+                                                 de[i],
+                                                 NULL,
+                                                 userdata);
+                                        if (r == RECURSE_DIR_LEAVE_DIRECTORY)
+                                                break;
+                                        if (!IN_SET(r, RECURSE_DIR_CONTINUE, RECURSE_DIR_SKIP_ENTRY))
+                                                return r;
+
+                                        continue;
+                                }
+
+                                /* If it's not a subdir, then let's handle it like a regular inode below */
+
+                        } else {
+                                /* If we managed to get a DIR* off the inode, it's definitely a directory. */
+                                de[i]->d_type = DT_DIR;
+
+                                if (statx_mask != 0 || (flags & RECURSE_DIR_SAME_MOUNT)) {
+                                        r = statx_fallback(dirfd(subdir), "", AT_EMPTY_PATH, statx_mask, &sx);
+                                        if (r < 0)
+                                                return r;
+
+                                        sx_valid = true;
+                                }
+                        }
+                }
+
+                if (!subdir) {
+                        /* It's not a subdirectory. */
+
+                        if (flags & RECURSE_DIR_INODE_FD) {
+
+                                inode_fd = openat(dirfd(d), de[i]->d_name, O_PATH|O_NOFOLLOW|O_CLOEXEC);
+                                if (inode_fd < 0) {
+                                        if (errno == ENOENT) /* Vanished by now, go for next file immediately */
+                                                continue;
+
+                                        log_debug_errno(errno, "Failed to open directory entry '%s': %m", p);
+
+                                        assert(errno <= RECURSE_DIR_SKIP_OPEN_INODE_ERROR_MAX - RECURSE_DIR_SKIP_OPEN_INODE_ERROR_BASE);
+
+                                        r = func(RECURSE_DIR_SKIP_OPEN_INODE_ERROR_BASE + errno,
+                                                 p,
+                                                 dirfd(d),
+                                                 -1,
+                                                 de[i],
+                                                 NULL,
+                                                 userdata);
+                                        if (r == RECURSE_DIR_LEAVE_DIRECTORY)
+                                                break;
+                                        if (!IN_SET(r, RECURSE_DIR_CONTINUE, RECURSE_DIR_SKIP_ENTRY))
+                                                return r;
+
+                                        continue;
+                                }
+
+                                /* If we open the inode, then verify it's actually a non-directory, like we
+                                 * assume. Let's guarantee that we never pass statx data of a directory where
+                                 * caller expects a non-directory */
+
+                                r = statx_fallback(inode_fd, "", AT_EMPTY_PATH, statx_mask | STATX_TYPE, &sx);
+                                if (r < 0)
+                                        return r;
+
+                                assert(sx.stx_mask & STATX_TYPE);
+                                sx_valid = true;
+
+                                if (S_ISDIR(sx.stx_mode)) {
+                                        /* What? It's a directory now? Then someone must have quickly
+                                         * replaced it. Let's handle that gracefully: convert it to a
+                                         * directory fd — which sould be riskless now that we pinned the
+                                         * inode. */
+
+                                        subdir = xopendirat(AT_FDCWD, FORMAT_PROC_FD_PATH(inode_fd), 0);
+                                        if (!subdir)
+                                                return -errno;
+
+                                        inode_fd = safe_close(inode_fd);
+                                }
+
+                        } else if (statx_mask != 0 || (de[i]->d_type == DT_UNKNOWN && (flags & RECURSE_DIR_ENSURE_TYPE))) {
+
+                                r = statx_fallback(dirfd(d), de[i]->d_name, AT_SYMLINK_NOFOLLOW, statx_mask | STATX_TYPE, &sx);
+                                if (r == -ENOENT) /* Vanished by now? Go for next file immediately */
+                                        continue;
+                                if (r < 0) {
+                                        log_debug_errno(r, "Failed to stat directory entry '%s': %m", p);
+
+                                        assert(errno <= RECURSE_DIR_SKIP_STAT_INODE_ERROR_MAX - RECURSE_DIR_SKIP_STAT_INODE_ERROR_BASE);
+
+                                        r = func(RECURSE_DIR_SKIP_STAT_INODE_ERROR_BASE + -r,
+                                                 p,
+                                                 dirfd(d),
+                                                 -1,
+                                                 de[i],
+                                                 NULL,
+                                                 userdata);
+                                        if (r == RECURSE_DIR_LEAVE_DIRECTORY)
+                                                break;
+                                        if (!IN_SET(r, RECURSE_DIR_CONTINUE, RECURSE_DIR_SKIP_ENTRY))
+                                                return r;
+
+                                        continue;
+                                }
+
+                                assert(sx.stx_mask & STATX_TYPE);
+                                sx_valid = true;
+
+                                if (S_ISDIR(sx.stx_mode)) {
+                                        /* So it suddenly is a directory, but we couldn't open it as such
+                                         * earlier?  That is weird, and probably means somebody is racing
+                                         * against us. We could of course retry and open it as a directory
+                                         * again, but the chance to win here is limited. Hence, let's
+                                         * propagate this as EISDIR error instead. That way we make this
+                                         * something that can be reasonably handled, even though we give the
+                                         * guarantee that RECURSE_DIR_ENTRY is strictly issued for
+                                         * non-directory dirents. */
+
+                                        log_debug_errno(r, "Non-directory entry '%s' suddenly became a directory: %m", p);
+
+                                        r = func(RECURSE_DIR_SKIP_STAT_INODE_ERROR_BASE + EISDIR,
+                                                 p,
+                                                 dirfd(d),
+                                                 -1,
+                                                 de[i],
+                                                 NULL,
+                                                 userdata);
+                                        if (r == RECURSE_DIR_LEAVE_DIRECTORY)
+                                                break;
+                                        if (!IN_SET(r, RECURSE_DIR_CONTINUE, RECURSE_DIR_SKIP_ENTRY))
+                                                return r;
+
+                                        continue;
+                                }
+                        }
+                }
+
+                if (sx_valid) {
+                        /* Copy over the data we acquired through statx() if we acquired any */
+                        if (sx.stx_mask & STATX_TYPE) {
+                                assert(!!subdir == !!S_ISDIR(sx.stx_mode));
+                                de[i]->d_type = stat_mode_to_dirent_type(sx.stx_mode);
+                        }
+
+                        if (sx.stx_mask & STATX_INO)
+                                de[i]->d_ino = sx.stx_ino;
+                }
+
+                if (subdir) {
+                        if (FLAGS_SET(flags, RECURSE_DIR_SAME_MOUNT)) {
+                                bool is_mount;
+
+                                if (sx_valid && FLAGS_SET(sx.stx_attributes_mask, STATX_ATTR_MOUNT_ROOT))
+                                        is_mount = FLAGS_SET(sx.stx_attributes, STATX_ATTR_MOUNT_ROOT);
+                                else {
+                                        r = fd_is_mount_point(dirfd(d), de[i]->d_name, 0);
+                                        if (r < 0)
+                                                log_debug_errno(r, "Failed to determine whether %s is a submount, assuming not: %m", p);
+
+                                        is_mount = r > 0;
+                                }
+
+                                if (is_mount) {
+                                        r = func(RECURSE_DIR_SKIP_MOUNT,
+                                                 p,
+                                                 dirfd(d),
+                                                 dirfd(subdir),
+                                                 de[i],
+                                                 statx_mask != 0 ? &sx : NULL, /* only pass sx if user asked for it */
+                                                 userdata);
+                                        if (r == RECURSE_DIR_LEAVE_DIRECTORY)
+                                                break;
+                                        if (!IN_SET(r, RECURSE_DIR_CONTINUE, RECURSE_DIR_SKIP_ENTRY))
+                                                return r;
+
+                                        continue;
+                                }
+                        }
+
+                        if (n_depth_max <= 1) {
+                                /* When we reached max depth, generate a special event */
+
+                                r = func(RECURSE_DIR_SKIP_DEPTH,
+                                         p,
+                                         dirfd(d),
+                                         dirfd(subdir),
+                                         de[i],
+                                         statx_mask != 0 ? &sx : NULL, /* only pass sx if user asked for it */
+                                         userdata);
+                                if (r == RECURSE_DIR_LEAVE_DIRECTORY)
+                                        break;
+                                if (!IN_SET(r, RECURSE_DIR_CONTINUE, RECURSE_DIR_SKIP_ENTRY))
+                                        return r;
+
+                                continue;
+                        }
+
+                        r = func(RECURSE_DIR_ENTER,
+                                 p,
+                                 dirfd(d),
+                                 dirfd(subdir),
+                                 de[i],
+                                 statx_mask != 0 ? &sx : NULL, /* only pass sx if user asked for it */
+                                 userdata);
+                        if (r == RECURSE_DIR_LEAVE_DIRECTORY)
+                                break;
+                        if (r == RECURSE_DIR_SKIP_ENTRY)
+                                continue;
+                        if (r != RECURSE_DIR_CONTINUE)
+                                return r;
+
+                        r = recurse_dir(subdir,
+                                        p,
+                                        statx_mask,
+                                        n_depth_max - 1,
+                                        flags,
+                                        func,
+                                        userdata);
+                        if (r != 0)
+                                return r;
+
+                        r = func(RECURSE_DIR_LEAVE,
+                                 p,
+                                 dirfd(d),
+                                 dirfd(subdir),
+                                 de[i],
+                                 statx_mask != 0 ? &sx : NULL, /* only pass sx if user asked for it */
+                                 userdata);
+                } else
+                        /* Non-directory inode */
+                        r = func(RECURSE_DIR_ENTRY,
+                                 p,
+                                 dirfd(d),
+                                 inode_fd,
+                                 de[i],
+                                 statx_mask != 0 ? &sx : NULL, /* only pass sx if user asked for it */
+                                 userdata);
+
+
+                if (r == RECURSE_DIR_LEAVE_DIRECTORY)
+                        break;
+                if (!IN_SET(r, RECURSE_DIR_SKIP_ENTRY, RECURSE_DIR_CONTINUE))
+                        return r;
+        }
+
+        return 0;
+}
+
+int recurse_dir_at(
+                int atfd,
+                const char *path,
+                unsigned statx_mask,
+                unsigned n_depth_max,
+                RecurseDirFlags flags,
+                recurse_dir_func_t func,
+                void *userdata) {
+
+        _cleanup_closedir_ DIR *d = NULL;
+
+        d = xopendirat(atfd, path, 0);
+        if (!d)
+                return -errno;
+
+        return recurse_dir(d, path, statx_mask, n_depth_max, flags, func, userdata);
+}
diff --git a/src/basic/recurse-dir.h b/src/basic/recurse-dir.h
new file mode 100644
index 0000000000..0605884fc0
--- /dev/null
+++ b/src/basic/recurse-dir.h
@@ -0,0 +1,75 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+#pragma once
+
+#include <dirent.h>
+#include <limits.h>
+
+#include "errno-list.h"
+#include "stat-util.h"
+#include "macro.h"
+
+typedef enum RecurseDirEvent {
+        RECURSE_DIR_ENTER,      /* only for dir inodes */
+        RECURSE_DIR_LEAVE,      /* only for dir inodes */
+        RECURSE_DIR_ENTRY,      /* only for non-dir inodes */
+        RECURSE_DIR_SKIP_MOUNT, /* only for dir inodes: when we don't descent into submounts */
+        RECURSE_DIR_SKIP_DEPTH, /* only for dir inodes: when we reached the max depth */
+
+        /* If we hit an error opening/stating an entry, then we'll fire a
+         * 'RECURSE_DIR_SKIP_{OPEN_DIR|OPEN_INODE|STAT_INODE}_ERROR_BASE + errno' event. In this case 'de'
+         * will be valid, but the statx data NULL and the inode fd -1. */
+        RECURSE_DIR_SKIP_OPEN_DIR_ERROR_BASE,
+        RECURSE_DIR_SKIP_OPEN_DIR_ERROR_MAX = RECURSE_DIR_SKIP_OPEN_DIR_ERROR_BASE + ERRNO_MAX,
+
+        RECURSE_DIR_SKIP_OPEN_INODE_ERROR_BASE,
+        RECURSE_DIR_SKIP_OPEN_INODE_ERROR_MAX = RECURSE_DIR_SKIP_OPEN_INODE_ERROR_BASE + ERRNO_MAX,
+
+        RECURSE_DIR_SKIP_STAT_INODE_ERROR_BASE,
+        RECURSE_DIR_SKIP_STAT_INODE_ERROR_MAX = RECURSE_DIR_SKIP_STAT_INODE_ERROR_BASE + ERRNO_MAX,
+
+        _RECURSE_DIR_EVENT_MAX,
+        _RECURSE_DIR_EVENT_INVALID = -EINVAL,
+} RecurseDirEvent;
+
+#define RECURSE_DIR_CONTINUE 0
+#define RECURSE_DIR_LEAVE_DIRECTORY INT_MIN
+#define RECURSE_DIR_SKIP_ENTRY (INT_MIN+1)
+
+/* Make sure that the negative errno range and these two special returns don't overlap */
+assert_cc(RECURSE_DIR_LEAVE_DIRECTORY < -ERRNO_MAX);
+assert_cc(RECURSE_DIR_SKIP_ENTRY < -ERRNO_MAX);
+
+/* Prototype for the callback function that is called whenever we enter or leave a dir inode, or find another dir entry. Return values are:
+ *
+ * RECURSE_DIR_CONTINUE (i.e. 0) → continue with next entry
+ * RECURSE_DIR_LEAVE_DIRECTORY   → leave current directory immediately, don't process further siblings
+ * RECURSE_DIR_SKIP_ENTRY        → skip this entry otherwise (only makes sense on RECURSE_DIR_ENTER)
+ * others                        → terminate iteration entirely, return the specified value (idea is that
+ *                                 < 0 indicates errors and > 0 indicates various forms of success)
+ */
+typedef int (*recurse_dir_func_t)(
+                RecurseDirEvent event,
+                const char *path,        /* Full non-normalized path, i.e. the path specified during recurise_dir() with what we found appended */
+                int dir_fd,              /* fd of the current dir */
+                int inode_fd,            /* fd of the current entry in the current dir (O_DIRECTORY if directory, and O_PATH otherwise, but only if RECURSE_DIR_INODE_FD was set) */
+                const struct dirent *de, /* directory entry (always valid) */
+                const struct statx *sx,  /* statx data (only if statx_mask was non-zero) */
+                void *userdata);
+
+typedef enum RecurseDirFlags {
+        /* Interpreted by readdir_all() */
+        RECURSE_DIR_SORT         = 1 << 0,  /* sort file directory entries before processing them */
+        RECURSE_DIR_IGNORE_DOT   = 1 << 1,  /* ignore all dot files ("." and ".." are always ignored) */
+
+        /* Interpreted by recurse_dir() */
+        RECURSE_DIR_ENSURE_TYPE  = 1 << 2,  /* guarantees that 'd_type' field of 'de' is not DT_UNKNOWN */
+        RECURSE_DIR_SAME_MOUNT   = 1 << 3,  /* skips over subdirectories that are submounts */
+        RECURSE_DIR_INODE_FD     = 1 << 4,  /* passes an opened inode fd (O_DIRECTORY fd in case of dirs, O_PATH otherwise) */
+} RecurseDirFlags;
+
+struct dirent** readdir_all_free(struct dirent **array);
+DEFINE_TRIVIAL_CLEANUP_FUNC(struct dirent **, readdir_all_free);
+int readdir_all(DIR *d, RecurseDirFlags flags, struct dirent ***ret);
+
+int recurse_dir(DIR *d, const char *path, unsigned statx_mask, unsigned n_depth_max, RecurseDirFlags flags, recurse_dir_func_t func, void *userdata);
+int recurse_dir_at(int atfd, const char *path, unsigned statx_mask, unsigned n_depth_max, RecurseDirFlags flags, recurse_dir_func_t func, void *userdata);
diff --git a/src/boot/bootctl.c b/src/boot/bootctl.c
index bb3627ed6e..60060cd60c 100644
--- a/src/boot/bootctl.c
+++ b/src/boot/bootctl.c
@@ -2,7 +2,6 @@
 
 #include <ctype.h>
 #include <errno.h>
-#include <ftw.h>
 #include <getopt.h>
 #include <limits.h>
 #include <linux/magic.h>
diff --git a/src/core/kmod-setup.c b/src/core/kmod-setup.c
index 8a7f82812a..d054668b8e 100644
--- a/src/core/kmod-setup.c
+++ b/src/core/kmod-setup.c
@@ -1,6 +1,5 @@
 /* SPDX-License-Identifier: LGPL-2.1-or-later */
 
-#include <ftw.h>
 #include <unistd.h>
 
 #include "alloc-util.h"
@@ -9,6 +8,7 @@
 #include "fileio.h"
 #include "kmod-setup.h"
 #include "macro.h"
+#include "recurse-dir.h"
 #include "string-util.h"
 
 #if HAVE_KMOD
@@ -28,39 +28,57 @@ static void systemd_kmod_log(
         REENABLE_WARNING;
 }
 
-static int has_virtio_rng_nftw_cb(
-                const char *fpath,
-                const struct stat *sb,
-                int tflag,
-                struct FTW *ftwbuf) {
+static int has_virtio_rng_recurse_dir_cb(
+                RecurseDirEvent event,
+                const char *path,
+                int dir_fd,
+                int inode_fd,
+                const struct dirent *de,
+                const struct statx *sx,
+                void *userdata) {
 
         _cleanup_free_ char *alias = NULL;
         int r;
 
-        if ((FTW_D == tflag) && (ftwbuf->level > 2))
-                return FTW_SKIP_SUBTREE;
+        if (event != RECURSE_DIR_ENTRY)
+                return RECURSE_DIR_CONTINUE;
 
-        if (FTW_F != tflag)
-                return FTW_CONTINUE;
+        if (de->d_type != DT_REG)
+                return RECURSE_DIR_CONTINUE;
 
-        if (!endswith(fpath, "/modalias"))
-                return FTW_CONTINUE;
+        if (!streq(de->d_name, "modalias"))
+                return RECURSE_DIR_CONTINUE;
 
-        r = read_one_line_file(fpath, &alias);
-        if (r < 0)
-                return FTW_SKIP_SIBLINGS;
+        r = read_one_line_file(path, &alias);
+        if (r < 0) {
+                log_debug_errno(r, "Failed to read %s, ignoring: %m", path);
+                return RECURSE_DIR_LEAVE_DIRECTORY;
+        }
 
         if (startswith(alias, "pci:v00001AF4d00001005"))
-                return FTW_STOP;
+                return 1;
 
         if (startswith(alias, "pci:v00001AF4d00001044"))
-                return FTW_STOP;
+                return 1;
 
-        return FTW_SKIP_SIBLINGS;
+        return RECURSE_DIR_LEAVE_DIRECTORY;
 }
 
 static bool has_virtio_rng(void) {
-        return (nftw("/sys/devices/pci0000:00", has_virtio_rng_nftw_cb, 64, FTW_MOUNT|FTW_PHYS|FTW_ACTIONRETVAL) == FTW_STOP);
+        int r;
+
+        r = recurse_dir_at(
+                        AT_FDCWD,
+                        "/sys/devices/pci0000:00",
+                        /* statx_mask= */ 0,
+                        /* n_depth_max= */ 2,
+                        RECURSE_DIR_ENSURE_TYPE,
+                        has_virtio_rng_recurse_dir_cb,
+                        NULL);
+        if (r < 0)
+                log_debug_errno(r, "Failed to determine whether host has virtio-rng device, ignoring: %m");
+
+        return r > 0;
 }
 #endif
 
diff --git a/src/locale/localectl.c b/src/locale/localectl.c
index 0de3532f97..7e63976dfc 100644
--- a/src/locale/localectl.c
+++ b/src/locale/localectl.c
@@ -1,6 +1,5 @@
 /* SPDX-License-Identifier: LGPL-2.1-or-later */
 
-#include <ftw.h>
 #include <getopt.h>
 #include <stdbool.h>
 #include <stdlib.h>
diff --git a/src/shared/cgroup-setup.c b/src/shared/cgroup-setup.c
index f197f715c7..7dee7e9512 100644
--- a/src/shared/cgroup-setup.c
+++ b/src/shared/cgroup-setup.c
@@ -1,22 +1,22 @@
 /* SPDX-License-Identifier: LGPL-2.1-or-later */
 
-#include <ftw.h>
 #include <unistd.h>
 
 #include "cgroup-setup.h"
 #include "cgroup-util.h"
 #include "errno-util.h"
+#include "fd-util.h"
+#include "fileio.h"
+#include "fs-util.h"
+#include "mkdir.h"
 #include "parse-util.h"
 #include "path-util.h"
 #include "proc-cmdline.h"
+#include "process-util.h"
+#include "recurse-dir.h"
 #include "stdio-util.h"
 #include "string-util.h"
-#include "fs-util.h"
-#include "mkdir.h"
-#include "process-util.h"
-#include "fileio.h"
 #include "user-util.h"
-#include "fd-util.h"
 
 bool cg_is_unified_wanted(void) {
         static thread_local int wanted = -1;
@@ -149,19 +149,23 @@ int cg_blkio_weight_parse(const char *s, uint64_t *ret) {
         return 0;
 }
 
-static int trim_cb(const char *path, const struct stat *sb, int typeflag, struct FTW *ftwbuf) {
-        assert(path);
-        assert(sb);
-        assert(ftwbuf);
-
-        if (typeflag != FTW_DP)
-                return 0;
-
-        if (ftwbuf->level < 1)
-                return 0;
-
-        (void) rmdir(path);
-        return 0;
+static int trim_cb(
+                RecurseDirEvent event,
+                const char *path,
+                int dir_fd,
+                int inode_fd,
+                const struct dirent *de,
+                const struct statx *sx,
+                void *userdata) {
+
+        /* Failures to delete inner cgroup we ignore (but debug log in case error code is unexpected) */
+        if (event == RECURSE_DIR_LEAVE &&
+            de->d_type == DT_DIR &&
+            unlinkat(dir_fd, de->d_name, AT_REMOVEDIR) < 0 &&
+            !IN_SET(errno, ENOENT, ENOTEMPTY, EBUSY))
+                log_debug_errno(errno, "Failed to trim inner cgroup %s, ignoring: %m", path);
+
+        return RECURSE_DIR_CONTINUE;
 }
 
 int cg_trim(const char *controller, const char *path, bool delete_root) {
@@ -169,32 +173,41 @@ int cg_trim(const char *controller, const char *path, bool delete_root) {
         int r, q;
 
         assert(path);
+        assert(controller);
 
         r = cg_get_path(controller, path, NULL, &fs);
         if (r < 0)
                 return r;
 
-        errno = 0;
-        if (nftw(fs, trim_cb, 64, FTW_DEPTH|FTW_MOUNT|FTW_PHYS) != 0) {
-                if (errno == ENOENT)
-                        r = 0;
-                else
-                        r = errno_or_else(EIO);
-        }
-
-        if (delete_root) {
-                if (rmdir(fs) < 0 && errno != ENOENT)
-                        return -errno;
+        r = recurse_dir_at(
+                        AT_FDCWD,
+                        fs,
+                        /* statx_mask= */ 0,
+                        /* n_depth_max= */ UINT_MAX,
+                        RECURSE_DIR_ENSURE_TYPE,
+                        trim_cb,
+                        NULL);
+        if (r == -ENOENT) /* non-existing is the ultimate trimming, hence no error */
+                r = 0;
+        else if (r < 0)
+                log_debug_errno(r, "Failed to iterate through cgroup %s: %m", path);
+
+        /* If we shall delete the top-level cgroup, then propagate the faiure to do so (except if it is
+         * already gone anyway). Also, let's debug log about this failure, except if the error code is an
+         * expected one. */
+        if (delete_root && !empty_or_root(path) &&
+            rmdir(fs) < 0 && errno != ENOENT) {
+                if (!IN_SET(errno, ENOTEMPTY, EBUSY))
+                        log_debug_errno(errno, "Failed to trim cgroup %s: %m", path);
+                if (r >= 0)
+                        r = -errno;
         }
 
         q = cg_hybrid_unified();
         if (q < 0)
                 return q;
-        if (q > 0 && streq(controller, SYSTEMD_CGROUP_CONTROLLER)) {
-                q = cg_trim(SYSTEMD_CGROUP_CONTROLLER_LEGACY, path, delete_root);
-                if (q < 0)
-                        log_warning_errno(q, "Failed to trim compat systemd cgroup %s: %m", path);
-        }
+        if (q > 0 && streq(controller, SYSTEMD_CGROUP_CONTROLLER))
+                (void) cg_trim(SYSTEMD_CGROUP_CONTROLLER_LEGACY, path, delete_root);
 
         return r;
 }
diff --git a/src/shared/kbd-util.c b/src/shared/kbd-util.c
index 92abaea65e..842ff3066b 100644
--- a/src/shared/kbd-util.c
+++ b/src/shared/kbd-util.c
@@ -1,86 +1,105 @@
 /* SPDX-License-Identifier: LGPL-2.1-or-later */
 
-#include <ftw.h>
-
 #include "errno-util.h"
 #include "kbd-util.h"
 #include "log.h"
 #include "nulstr-util.h"
 #include "path-util.h"
+#include "recurse-dir.h"
 #include "set.h"
 #include "string-util.h"
 #include "strv.h"
 #include "utf8.h"
 
-static thread_local const char *keymap_name = NULL;
-static thread_local Set *keymaps = NULL;
-
-static int nftw_cb(
-                const char *fpath,
-                const struct stat *sb,
-                int tflag,
-                struct FTW *ftwbuf) {
-
+struct recurse_dir_userdata {
+        const char *keymap_name;
+        Set *keymaps;
+};
+
+static int keymap_recurse_dir_callback(
+                RecurseDirEvent event,
+                const char *path,
+                int dir_fd,
+                int inode_fd,
+                const struct dirent *de,
+                const struct statx *sx,
+                void *userdata) {
+
+        struct recurse_dir_userdata *data = userdata;
         _cleanup_free_ char *p = NULL;
         int r;
 
-        /* If keymap_name is non-null, return true if keymap keymap_name is found.
-         * Otherwise, add all keymaps to keymaps. */
+        assert(de);
 
-        if (tflag != FTW_F)
-                return 0;
+        /* If 'keymap_name' is non-NULL, return true if keymap 'keymap_name' is found.  Otherwise, add all
+         * keymaps to 'keymaps'. */
+
+        if (event != RECURSE_DIR_ENTRY)
+                return RECURSE_DIR_CONTINUE;
 
-        fpath = basename(fpath);
+        if (!IN_SET(de->d_type, DT_REG, DT_LNK))
+                return RECURSE_DIR_CONTINUE;
 
-        const char *e = endswith(fpath, ".map") ?: endswith(fpath, ".map.gz");
+        const char *e = endswith(de->d_name, ".map") ?: endswith(de->d_name, ".map.gz");
         if (!e)
-                return 0;
+                return RECURSE_DIR_CONTINUE;
 
-        p = strndup(fpath, e - fpath);
-        if (!p) {
-                errno = ENOMEM;
-                return -1;
-        }
+        p = strndup(de->d_name, e - de->d_name);
+        if (!p)
+                return -ENOMEM;
+
+        if (data->keymap_name)
+                return streq(p, data->keymap_name) ? 1 : RECURSE_DIR_CONTINUE;
 
-        if (keymap_name)
-                return streq(p, keymap_name);
+        assert(data->keymaps);
 
         if (!keymap_is_valid(p))
                 return 0;
 
-        r = set_consume(keymaps, TAKE_PTR(p));
-        if (r < 0 && r != -EEXIST) {
-                errno = -r;
-                return -1;
-        }
+        r = set_consume(data->keymaps, TAKE_PTR(p));
+        if (r < 0)
+                return r;
 
-        return 0;
+        return RECURSE_DIR_CONTINUE;
 }
 
 int get_keymaps(char ***ret) {
+        _cleanup_(set_free_freep) Set *keymaps = NULL;
+        int r;
+
         keymaps = set_new(&string_hash_ops);
         if (!keymaps)
                 return -ENOMEM;
 
         const char *dir;
-        NULSTR_FOREACH(dir, KBD_KEYMAP_DIRS)
-                if (nftw(dir, nftw_cb, 20, FTW_PHYS) < 0) {
-                        if (errno == ENOENT)
+        NULSTR_FOREACH(dir, KBD_KEYMAP_DIRS) {
+                r = recurse_dir_at(
+                                AT_FDCWD,
+                                dir,
+                                /* statx_mask= */ 0,
+                                /* n_depth_max= */ UINT_MAX,
+                                RECURSE_DIR_IGNORE_DOT|RECURSE_DIR_ENSURE_TYPE,
+                                keymap_recurse_dir_callback,
+                                &(struct recurse_dir_userdata) {
+                                        .keymaps = keymaps,
+                                });
+                if (r < 0) {
+                        if (r == -ENOENT)
                                 continue;
-                        if (ERRNO_IS_RESOURCE(errno)) {
-                                keymaps = set_free_free(keymaps);
-                                return log_warning_errno(errno, "Failed to read keymap list from %s: %m", dir);
-                        }
-                        log_debug_errno(errno, "Failed to read keymap list from %s, ignoring: %m", dir);
+                        if (ERRNO_IS_RESOURCE(r))
+                                return log_warning_errno(r, "Failed to read keymap list from %s: %m", dir);
+
+                        log_debug_errno(r, "Failed to read keymap list from %s, ignoring: %m", dir);
                 }
+        }
 
         _cleanup_strv_free_ char **l = set_get_strv(keymaps);
-        if (!l) {
-                keymaps = set_free_free(keymaps);
+        if (!l)
                 return -ENOMEM;
-        }
 
-        keymaps = set_free(keymaps);
+        keymaps = set_free(keymaps); /* If we got the strv above, then do a set_free() rather than
+                                      * set_free_free() since the entries of the set are now owned by the
+                                      * strv */
 
         if (strv_isempty(l))
                 return -ENOENT;
@@ -88,7 +107,6 @@ int get_keymaps(char ***ret) {
         strv_sort(l);
 
         *ret = TAKE_PTR(l);
-
         return 0;
 }
 
@@ -117,18 +135,29 @@ int keymap_exists(const char *name) {
         if (!keymap_is_valid(name))
                 return -EINVAL;
 
-        keymap_name = name;
-
         const char *dir;
         NULSTR_FOREACH(dir, KBD_KEYMAP_DIRS) {
-                r = nftw(dir, nftw_cb, 20, FTW_PHYS);
+                r = recurse_dir_at(
+                                AT_FDCWD,
+                                dir,
+                                /* statx_mask= */ 0,
+                                /* n_depth_max= */ UINT_MAX,
+                                RECURSE_DIR_IGNORE_DOT|RECURSE_DIR_ENSURE_TYPE,
+                                keymap_recurse_dir_callback,
+                                &(struct recurse_dir_userdata) {
+                                        .keymap_name = name,
+                                });
+                if (r == -ENOENT)
+                        continue;
+                if (ERRNO_IS_RESOURCE(r))
+                        return r;
+                if (r < 0) {
+                        log_debug_errno(r, "Failed to read keymap list from %s, ignoring: %m", dir);
+                        continue;
+                }
                 if (r > 0)
                         break;
-                if (r < 0 && errno != ENOENT)
-                        log_debug_errno(errno, "Failed to read keymap list from %s, ignoring: %m", dir);
         }
 
-        keymap_name = NULL;
-
         return r > 0;
 }
diff --git a/src/shared/mount-setup.c b/src/shared/mount-setup.c
index ae148541d7..7057a8763e 100644
--- a/src/shared/mount-setup.c
+++ b/src/shared/mount-setup.c
@@ -1,7 +1,6 @@
 /* SPDX-License-Identifier: LGPL-2.1-or-later */
 
 #include <errno.h>
-#include <ftw.h>
 #include <stdlib.h>
 #include <sys/mount.h>
 #include <sys/statvfs.h>
@@ -9,9 +8,9 @@
 
 #include "alloc-util.h"
 #include "bus-util.h"
+#include "cgroup-setup.h"
 #include "cgroup-util.h"
 #include "conf-files.h"
-#include "cgroup-setup.h"
 #include "dev-setup.h"
 #include "dirent-util.h"
 #include "efi-loader.h"
@@ -27,6 +26,7 @@
 #include "mountpoint-util.h"
 #include "nulstr-util.h"
 #include "path-util.h"
+#include "recurse-dir.h"
 #include "set.h"
 #include "smack-util.h"
 #include "strv.h"
@@ -365,27 +365,46 @@ int mount_cgroup_controllers(void) {
 }
 
 #if HAVE_SELINUX || ENABLE_SMACK
-static int nftw_cb(
-                const char *fpath,
-                const struct stat *sb,
-                int tflag,
-                struct FTW *ftwbuf) {
-
-        /* No need to label /dev twice in a row... */
-        if (_unlikely_(ftwbuf->level == 0))
-                return FTW_CONTINUE;
-
-        (void) label_fix(fpath, 0);
-
-        /* /run/initramfs is static data and big, no need to
-         * dynamically relabel its contents at boot... */
-        if (_unlikely_(ftwbuf->level == 1 &&
-                      tflag == FTW_D &&
-                      streq(fpath, "/run/initramfs")))
-                return FTW_SKIP_SUBTREE;
-
-        return FTW_CONTINUE;
-};
+static int relabel_cb(
+                RecurseDirEvent event,
+                const char *path,
+                int dir_fd,
+                int inode_fd,
+                const struct dirent *de,
+                const struct statx *sx,
+                void *userdata) {
+
+        switch (event) {
+
+        case RECURSE_DIR_LEAVE:
+        case RECURSE_DIR_SKIP_MOUNT:
+                /* If we already saw this dirent when entering it or this is a dirent that on a different
+                 * mount, don't relabel it. */
+                return RECURSE_DIR_CONTINUE;
+
+        case RECURSE_DIR_ENTER:
+                /* /run/initramfs is static data and big, no need to dynamically relabel its contents at boot... */
+                if (path_equal(path, "/run/initramfs"))
+                        return RECURSE_DIR_SKIP_ENTRY;
+
+                _fallthrough_;
+
+        default:
+                /* Otherwise, label it, even if we had trouble stat()ing it and similar. SELinux can figure this out */
+                (void) label_fix(path, 0);
+                return RECURSE_DIR_CONTINUE;
+        }
+}
+
+static int relabel_tree(const char *path) {
+        int r;
+
+        r = recurse_dir_at(AT_FDCWD, path, 0, UINT_MAX, RECURSE_DIR_ENSURE_TYPE|RECURSE_DIR_SAME_MOUNT, relabel_cb, NULL);
+        if (r < 0)
+                log_debug_errno(r, "Failed to recursively relabel '%s': %m", path);
+
+        return r;
+}
 
 static int relabel_cgroup_filesystems(void) {
         int r;
@@ -404,7 +423,7 @@ static int relabel_cgroup_filesystems(void) {
                         (void) mount_nofollow(NULL, "/sys/fs/cgroup", NULL, MS_REMOUNT, NULL);
 
                 (void) label_fix("/sys/fs/cgroup", 0);
-                (void) nftw("/sys/fs/cgroup", nftw_cb, 64, FTW_MOUNT|FTW_PHYS|FTW_ACTIONRETVAL);
+                (void) relabel_tree("/sys/fs/cgroup");
 
                 if (st.f_flags & ST_RDONLY)
                         (void) mount_nofollow(NULL, "/sys/fs/cgroup", NULL, MS_REMOUNT|MS_RDONLY, NULL);
@@ -468,7 +487,7 @@ static int relabel_extra(void) {
 
                         log_debug("Relabelling additional file/directory '%s'.", line);
                         (void) label_fix(line, 0);
-                        (void) nftw(line, nftw_cb, 64, FTW_MOUNT|FTW_PHYS|FTW_ACTIONRETVAL);
+                        (void) relabel_tree(line);
                         c++;
                 }
 
@@ -505,7 +524,7 @@ int mount_setup(bool loaded_policy, bool leave_propagation) {
                 before_relabel = now(CLOCK_MONOTONIC);
 
                 FOREACH_STRING(i, "/dev", "/dev/shm", "/run")
-                        (void) nftw(i, nftw_cb, 64, FTW_MOUNT|FTW_PHYS|FTW_ACTIONRETVAL);
+                        (void) relabel_tree(i);
 
                 (void) relabel_cgroup_filesystems();
 
diff --git a/src/test/meson.build b/src/test/meson.build
index cef2532407..fea7f107fd 100644
--- a/src/test/meson.build
+++ b/src/test/meson.build
@@ -148,6 +148,8 @@ tests += [
 
         [['src/test/test-utf8.c']],
 
+        [['src/test/test-kbd-util.c']],
+
         [['src/test/test-blockdev-util.c']],
 
         [['src/test/test-dev-setup.c']],
@@ -163,6 +165,8 @@ tests += [
 
         [['src/test/test-copy.c']],
 
+        [['src/test/test-recurse-dir.c']],
+
         [['src/test/test-data-fd-util.c']],
 
         [['src/test/test-static-destruct.c']],
diff --git a/src/test/test-kbd-util.c b/src/test/test-kbd-util.c
new file mode 100644
index 0000000000..c7b4b842c0
--- /dev/null
+++ b/src/test/test-kbd-util.c
@@ -0,0 +1,28 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include "kbd-util.h"
+#include "log.h"
+#include "strv.h"
+#include "tests.h"
+
+int main(int argc, char *argv[]) {
+        _cleanup_strv_free_ char **maps = NULL;
+        char **m;
+        int r;
+
+        log_show_color(true);
+        test_setup_logging(LOG_DEBUG);
+
+        r = get_keymaps(&maps);
+        if (r < 0) {
+                log_error_errno(r, "Failed to aquire keymaps: %m");
+                return 0;
+        }
+
+        STRV_FOREACH(m, maps) {
+                log_info("Found keymap: %s", *m);
+                assert_se(keymap_exists(*m) > 0);
+        }
+
+        return 0;
+}
diff --git a/src/test/test-recurse-dir.c b/src/test/test-recurse-dir.c
new file mode 100644
index 0000000000..5872936a77
--- /dev/null
+++ b/src/test/test-recurse-dir.c
@@ -0,0 +1,161 @@
+/* SPDX-License-Identifier: LGPL-2.1-or-later */
+
+#include <ftw.h>
+
+#include "log.h"
+#include "recurse-dir.h"
+#include "strv.h"
+#include "tests.h"
+
+static char **list_nftw = NULL;
+
+static int nftw_cb(
+                const char *fpath,
+                const struct stat *sb,
+                int typeflag,
+                struct FTW *ftwbuf) {
+
+        if (ftwbuf->level == 0) /* skip top-level */
+                return FTW_CONTINUE;
+
+        switch (typeflag) {
+
+        case FTW_F:
+                log_debug("ftw found %s", fpath);
+                assert_se(strv_extend(&list_nftw, fpath) >= 0);
+                break;
+
+        case FTW_SL:
+                log_debug("ftw found symlink %s", fpath);
+                assert_se(strv_extendf(&list_nftw, "%s→", fpath) >= 0);
+                break;
+
+        case FTW_D:
+                log_debug("ftw entering %s", fpath);
+                assert_se(strv_extendf(&list_nftw, "%s/", fpath) >= 0);
+                break;
+
+        case FTW_DNR:
+                log_debug("ftw open directory failed %s", fpath);
+                break;
+
+        case FTW_NS:
+                log_debug("ftw stat inode failed %s", fpath);
+                break;
+
+        case FTW_DP:
+        case FTW_SLN:
+        default:
+                assert_not_reached();
+        }
+
+        return FTW_CONTINUE;
+}
+
+static int recurse_dir_callback(
+                RecurseDirEvent event,
+                const char *path,
+                int dir_fd,
+                int inode_fd,
+                const struct dirent *de,
+                const struct statx *sx,
+                void *userdata) {
+
+        char ***l = userdata;
+
+        assert(path);
+        assert(de);
+
+        switch (event) {
+
+        case RECURSE_DIR_ENTRY:
+                assert_se(!IN_SET(de->d_type, DT_UNKNOWN, DT_DIR));
+
+                log_debug("found %s", path);
+
+                if (de->d_type == DT_LNK)
+                        assert_se(strv_extendf(l, "%s→", path) >= 0);
+                else
+                        assert_se(strv_extend(l, path) >= 0);
+                break;
+
+        case RECURSE_DIR_ENTER:
+                assert_se(de->d_type == DT_DIR);
+
+                log_debug("entering %s", path);
+                assert_se(strv_extendf(l, "%s/", path) >= 0);
+                break;
+
+        case RECURSE_DIR_LEAVE:
+                log_debug("leaving %s", path);
+                break;
+
+        case RECURSE_DIR_SKIP_MOUNT:
+                log_debug("skipping mount %s", path);
+                break;
+
+        case RECURSE_DIR_SKIP_DEPTH:
+                log_debug("skipping depth %s", path);
+                break;
+
+        case RECURSE_DIR_SKIP_OPEN_DIR_ERROR_BASE...RECURSE_DIR_SKIP_OPEN_DIR_ERROR_MAX:
+                log_debug_errno(event - RECURSE_DIR_SKIP_OPEN_DIR_ERROR_BASE, "failed to open dir %s: %m", path);
+                break;
+
+        case RECURSE_DIR_SKIP_OPEN_INODE_ERROR_BASE...RECURSE_DIR_SKIP_OPEN_INODE_ERROR_MAX:
+                log_debug_errno(event - RECURSE_DIR_SKIP_OPEN_INODE_ERROR_BASE, "failed to open inode %s: %m", path);
+                break;
+
+        case RECURSE_DIR_SKIP_STAT_INODE_ERROR_BASE...RECURSE_DIR_SKIP_STAT_INODE_ERROR_MAX:
+                log_debug_errno(event - RECURSE_DIR_SKIP_STAT_INODE_ERROR_BASE, "failed to stat inode %s: %m", path);
+                break;
+
+        default:
+                assert_not_reached();
+        }
+
+        return RECURSE_DIR_CONTINUE;
+}
+
+int main(int argc, char *argv[]) {
+        _cleanup_strv_free_ char **list_recurse_dir = NULL;
+        const char *p;
+        int r;
+
+        log_show_color(true);
+        test_setup_logging(LOG_INFO);
+
+        if (argc > 1)
+                p = argv[1];
+        else
+                p = "/usr/share/man"; /* something hopefully reasonably stable while we run (and limited in size) */
+
+        /* Enumerate the specified dirs in full, once via nftw(), and once via recurse_dir(), and ensure the results are identical */
+        r = recurse_dir_at(AT_FDCWD, p, 0, UINT_MAX, RECURSE_DIR_SORT|RECURSE_DIR_ENSURE_TYPE|RECURSE_DIR_SAME_MOUNT, recurse_dir_callback, &list_recurse_dir);
+        if (r == -ENOENT) {
+                log_warning_errno(r, "Couldn't open directory %s, ignoring: %m", p);
+                return EXIT_TEST_SKIP;
+        }
+        assert_se(r >= 0);
+
+        assert_se(nftw(p, nftw_cb, 64, FTW_PHYS|FTW_MOUNT) >= 0);
+
+        strv_sort(list_recurse_dir);
+        strv_sort(list_nftw);
+
+        for (size_t i = 0;; i++) {
+                const char *a = list_nftw ? list_nftw[i] : NULL,
+                        *b = list_recurse_dir ? list_recurse_dir[i] : NULL;
+
+                if (!streq_ptr(a, b)) {
+                        log_error("entry %zu different: %s vs %s", i, strna(a), strna(b));
+                        assert_not_reached();
+                }
+
+                if (!a)
+                        break;
+        }
+
+        list_nftw = strv_free(list_nftw);
+        return 0;
+}