dissect: add --manifest switch to generate a UAPI.16 manifest file from a directory tree

The UAPI.16 spec is being discussed here:

uapi-group/specifications#213

Author: poettering

src/dissect/dissect.c

@@ -49,6 +49,7 @@
 #include "pretty-print.h"
 #include "process-util.h"
 #include "recurse-dir.h"
+#include "rm-rf.h"
 #include "runtime-scope.h"
 #include "sha256.h"
 #include "shift-uid.h"
@@ -70,6 +71,7 @@ static enum {
         ACTION_DETACH,
         ACTION_LIST,
         ACTION_MTREE,
+        ACTION_MANIFEST,
         ACTION_WITH,
         ACTION_COPY_FROM,
         ACTION_COPY_TO,
@@ -150,6 +152,7 @@ static int help(void) {
                "%1$s [OPTIONS...] --detach PATH\n"
                "%1$s [OPTIONS...] --list IMAGE\n"
                "%1$s [OPTIONS...] --mtree IMAGE\n"
+               "%1$s [OPTIONS...] --manifest IMAGE\n"
                "%1$s [OPTIONS...] --with IMAGE [COMMAND…]\n"
                "%1$s [OPTIONS...] --copy-from IMAGE PATH [TARGET]\n"
                "%1$s [OPTIONS...] --copy-to IMAGE [SOURCE] PATH\n"
@@ -478,6 +481,11 @@ static int parse_argv(int argc, char *argv[]) {
                         arg_flags |= DISSECT_IMAGE_READ_ONLY;
                         break;
 
+                OPTION_LONG("manifest", NULL, "Show UAPI.16 manifest of OS image"):
+                        arg_action = ACTION_MANIFEST;
+                        arg_flags |= DISSECT_IMAGE_READ_ONLY;
+                        break;
+
                 OPTION_FULL(OPTION_STOPS_PARSING, /* sc= */ 0, "with", NULL, "Mount, run command, unmount"):
                         arg_action = ACTION_WITH;
                         break;
@@ -581,6 +589,7 @@ static int parse_argv(int argc, char *argv[]) {
 
         case ACTION_LIST:
         case ACTION_MTREE:
+        case ACTION_MANIFEST:
                 if (strv_length(args) != 1)
                         return log_error_errno(SYNTHETIC_ERRNO(EINVAL),
                                                "Expected an image file or directory path as only argument.");
@@ -1328,13 +1337,272 @@ static int mtree_print_item(
         return RECURSE_DIR_CONTINUE;
 }
 
-static int action_list_or_mtree_or_copy_or_make_archive(DissectedImage *m, LoopDevice *d, int userns_fd) {
+typedef struct HardlinkData {
+        int db_fd;
+        char *db_path;
+        uint64_t counter;
+} HardlinkData;
+
+static void hardlink_data_done(HardlinkData *hd) {
+        assert(hd);
+
+        if (hd->db_fd >= 0)
+                (void) rm_rf_children(TAKE_FD(hd->db_fd), REMOVE_PHYSICAL|REMOVE_REWIND, /* root_dev= */ NULL);
+
+        if (hd->db_path)
+                hd->db_path = rmdir_and_free(hd->db_path);
+}
+
+static int hardlink_data_lookup(
+                HardlinkData *hd,
+                const char *path,
+                int inode_fd,
+                const struct statx *sx,
+                uint64_t *ret_hardlink_counter,
+                uint8_t ret_hash[static SHA256_DIGEST_SIZE],
+                bool *ret_hash_valid) {
+
+        _cleanup_free_ struct file_handle *handle = NULL;
+        _cleanup_free_ char *slt = NULL, *fhs = NULL;
+        int r;
+
+        assert(hd);
+        assert(sx);
+        assert(ret_hardlink_counter);
+        assert(ret_hash);
+        assert(ret_hash_valid);
+
+        if (!FLAGS_SET(sx->stx_mask, STATX_NLINK|STATX_TYPE))
+                goto nope;
+
+        if (S_ISDIR(sx->stx_mode) || sx->stx_nlink <= 1)
+                goto nope;
+
+        if (inode_fd < 0)
+                goto nope;
+
+        /* This is a hardlinked inode, apparently. Let's figure out its inode token */
+
+        /* Generate a unique identifier for this inode via the kernel's FID logic. */
+        uint64_t unique_mnt_id = 0;
+        int mnt_id = 0;
+        r = name_to_handle_at_try_fid(inode_fd, /* path= */ NULL, &handle, &mnt_id, &unique_mnt_id, AT_EMPTY_PATH);
+        if (ERRNO_IS_NEG_NOT_SUPPORTED(r)) {
+                log_debug_errno(r, "File system does not support FIDs, not checking for hardlinks: %m");
+                goto nope;
+        }
+        if (r < 0)
+                return log_error_errno(r, "Failed to acquire FID of path: %m");
+
+        struct sha256_ctx ctx;
+        sha256_init_ctx(&ctx);
+        sha256_process_bytes(&handle->handle_bytes, sizeof(handle->handle_bytes), &ctx);
+        sha256_process_bytes(&handle->handle_type, sizeof(handle->handle_type), &ctx);
+        sha256_process_bytes(&handle->f_handle, handle->handle_bytes, &ctx);
+        sha256_process_bytes(&mnt_id, sizeof(mnt_id), &ctx);
+        sha256_process_bytes(&unique_mnt_id, sizeof(unique_mnt_id), &ctx);
+
+        uint8_t fid_hash[SHA256_DIGEST_SIZE];
+        sha256_finish_ctx(&ctx, fid_hash);
+
+        fhs = hexmem(fid_hash, sizeof(fid_hash));
+        if (!fhs)
+                return log_oom();
+
+        bool is_empty = false;
+        if (hd->db_fd < 0) {
+                assert(!hd->db_path);
+
+                /* Let's use a temporary directory on disk as hardlink database, so that we can
+                 * process very large images without allocating ridiculous amounts of memory. */
+
+                const char *vtd = NULL;
+                r = var_tmp_dir(&vtd);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to determine /var/tmp/ directory: %m");
+
+                _cleanup_free_ char *template = path_join(vtd, "dissect-hardlink-db-XXXXXX");
+                if (!template)
+                        return log_oom();
+
+                hd->db_fd = mkdtemp_open(template, /* flags= */ 0, &hd->db_path);
+                if (hd->db_fd < 0)
+                        return log_error_errno(hd->db_fd, "Failed to create temporary directory: %m");
+
+                is_empty = true;
+        }
+
+        uint64_t hardlink_counter;
+        uint8_t hash[SHA256_DIGEST_SIZE];
+        bool have_hash = false;
+
+        r = is_empty ? -ENOENT : readlinkat_malloc(hd->db_fd, fhs, &slt);
+        if (r < 0) {
+                if (r != -ENOENT)
+                        return log_error_errno(r, "Failed to read hardlink database symlink '%s': %m", fhs);
+
+                if (hd->counter >= UINT64_MAX) /* Paranoia */
+                        return log_error_errno(SYNTHETIC_ERRNO(ERANGE), "Hardlink overflow.");
+
+                hardlink_counter = ++hd->counter;
+
+                slt = asprintf_safe("%" PRIu64, hardlink_counter);
+                if (!slt)
+                        return log_oom();
+
+                if (S_ISREG(sx->stx_mode) && arg_mtree_hash) {
+                        r = get_file_sha256(inode_fd, hash);
+                        if (r < 0)
+                                log_warning_errno(r, "Failed to calculate file SHA256 sum for '%s', ignoring: %m", empty_to_root(path));
+                        else {
+                                _cleanup_free_ char *hs = hexmem(hash, sizeof(hash));
+                                if (!hs)
+                                        return log_oom();
+
+                                if (!strextend(&slt, ":", hs))
+                                        return log_oom();
+
+                                have_hash = true;
+                        }
+                }
+
+                if (symlinkat(slt, hd->db_fd, fhs) < 0)
+                        return log_error_errno(errno, "Failed to create hardlink database symlink '%s': %m", fhs);
+        } else {
+                char *colon = strchr(slt, ':');
+
+                if (colon) {
+                        _cleanup_free_ void *unhexxed = NULL;
+                        size_t unhexxed_size = 0;
+                        r = unhexmem(colon + 1, &unhexxed, &unhexxed_size);
+                        if (r < 0)
+                                return log_error_errno(r, "Failed to unhex file hash: %m");
+                        if (unhexxed_size != sizeof(hash))
+                                return log_error_errno(SYNTHETIC_ERRNO(EINVAL), "Malformed hash entry: %m");
+
+                        memcpy(hash, unhexxed, unhexxed_size);
+                        have_hash = true;
+                        *colon = 0;
+                }
+
+                r = safe_atou64(slt, &hardlink_counter);
+                if (r < 0)
+                        return log_error_errno(r, "Failed to decode hardlink counter: %m");
+        }
+
+        *ret_hardlink_counter = hardlink_counter;
+        *ret_hash_valid = have_hash;
+        if (have_hash)
+                memcpy(ret_hash, hash, sizeof(hash));
+        return 1;
+
+nope:
+        *ret_hardlink_counter = 0;
+        *ret_hash_valid = false;
+        return 0;
+}
+
+static int manifest_print_item(
+                RecurseDirEvent event,
+                const char *path,
+                int dir_fd,
+                int inode_fd,
+                const struct dirent *de,
+                const struct statx *sx,
+                void *userdata) {
+
+        HardlinkData *hd = ASSERT_PTR(userdata);
+        int r;
+
+        /* Generates a UAPI.16 Manifest of the directory tree */
+
+        if (!IN_SET(event, RECURSE_DIR_ENTER, RECURSE_DIR_ENTRY))
+                return RECURSE_DIR_CONTINUE;
+
+        bool is_root = isempty(path);
+
+        /* We ignore this to avoid input-as-output cycles (only on top-level though!) */
+        if (!is_root && (streq(path, "Uapi16Manifest") || startswith(path, "Uapi16Manifest.")))
+                return RECURSE_DIR_CONTINUE;
+
+        _cleanup_free_ char *data = NULL;
+        uint8_t hash[SHA256_DIGEST_SIZE];
+        uint64_t size = UINT64_MAX;
+        bool have_size = false, have_hash = false;
+        const char *type = NULL;
+        uint64_t hardlink_counter = 0;
+
+        r = hardlink_data_lookup(hd, path, inode_fd, sx, &hardlink_counter, hash, &have_hash);
+        if (r < 0)
+                return r;
+
+        if (FLAGS_SET(sx->stx_mask, STATX_TYPE)) {
+                switch (sx->stx_mode & S_IFMT) {
+
+                case S_IFREG:
+                        size = sx->stx_size;
+                        have_size = true;
+
+                        if (!have_hash && arg_mtree_hash && inode_fd >= 0) {
+                                r = get_file_sha256(inode_fd, hash);
+                                if (r < 0)
+                                        log_warning_errno(r, "Failed to calculate file SHA256 sum for '%s', ignoring: %m", empty_to_root(path));
+                                else
+                                        have_hash = true;
+                        }
+
+                        break;
+
+                case S_IFLNK:
+                        if (inode_fd >= 0) {
+                                r = readlinkat_malloc(inode_fd, "", &data);
+                                if (r < 0)
+                                        log_warning_errno(r, "Failed to read symlink '%s', ignoring: %m", empty_to_root(path));
+                                else {
+                                        size = strlen(data);
+                                        have_size = true;
+                                }
+                        }
+
+                        break;
+                }
+
+                /* Suppress the type on the top-level entry and on regular files. */
+                type = !is_root && !S_ISREG(sx->stx_mode) ? inode_type_to_string(sx->stx_mode) : NULL;
+        }
+
+        _cleanup_(sd_json_variant_unrefp) sd_json_variant *j = NULL;
+        r = sd_json_buildo(
+                        &j,
+                        SD_JSON_BUILD_PAIR_CONDITION(is_root, "mediaType", JSON_BUILD_CONST_STRING("application/vnd.uapi.16.manifest")),
+                        SD_JSON_BUILD_PAIR_CONDITION(!is_root, "name", SD_JSON_BUILD_STRING(path)),
+                        SD_JSON_BUILD_PAIR_CONDITION(!!type, "type", SD_JSON_BUILD_STRING(type)),
+                        SD_JSON_BUILD_PAIR_CONDITION(have_size, "size", SD_JSON_BUILD_UNSIGNED(size)),
+                        SD_JSON_BUILD_PAIR_CONDITION(!!data, "contents", SD_JSON_BUILD_ARRAY(SD_JSON_BUILD_OBJECT(SD_JSON_BUILD_PAIR("literal", SD_JSON_BUILD_BASE64(data, size))))),
+                        SD_JSON_BUILD_PAIR_CONDITION(have_hash, "sha256", SD_JSON_BUILD_HEX(hash, sizeof(hash))),
+                        SD_JSON_BUILD_PAIR_CONDITION(FLAGS_SET(sx->stx_mask, STATX_TYPE) && (S_ISBLK(sx->stx_mode) || S_ISCHR(sx->stx_mode)), "major", SD_JSON_BUILD_UNSIGNED(sx->stx_rdev_major)),
+                        SD_JSON_BUILD_PAIR_CONDITION(FLAGS_SET(sx->stx_mask, STATX_TYPE) && (S_ISBLK(sx->stx_mode) || S_ISCHR(sx->stx_mode)), "minor", SD_JSON_BUILD_UNSIGNED(sx->stx_rdev_minor)),
+                        SD_JSON_BUILD_PAIR_CONDITION(FLAGS_SET(sx->stx_mask, STATX_TYPE|STATX_MODE) && !S_ISLNK(sx->stx_mode), "mode", SD_JSON_BUILD_UNSIGNED(sx->stx_mode & 07777)),
+                        SD_JSON_BUILD_PAIR_CONDITION(FLAGS_SET(sx->stx_mask, STATX_UID), "uid", SD_JSON_BUILD_UNSIGNED(sx->stx_uid)),
+                        SD_JSON_BUILD_PAIR_CONDITION(FLAGS_SET(sx->stx_mask, STATX_GID), "gid", SD_JSON_BUILD_UNSIGNED(sx->stx_gid)),
+                        SD_JSON_BUILD_PAIR_CONDITION(hardlink_counter > 0, "inodeToken", SD_JSON_BUILD_UNSIGNED(hardlink_counter)));
+        if (r < 0)
+                return log_error_errno(r, "Failed to build JSON object: %m");
+
+        r = sd_json_variant_dump(j, arg_json_format_flags | SD_JSON_FORMAT_SEQ | SD_JSON_FORMAT_FLUSH, stdout, /* prefix= */ NULL);
+        if (r < 0)
+                return log_error_errno(r, "Failed to write JSON output: %m");
+
+        return RECURSE_DIR_CONTINUE;
+}
+
+static int action_list_or_mtree_or_manifest_or_copy_or_make_archive(DissectedImage *m, LoopDevice *d, int userns_fd) {
         _cleanup_(umount_and_freep) char *mounted_dir = NULL;
         _cleanup_free_ char *t = NULL;
         const char *root;
         int r;
 
-        assert(IN_SET(arg_action, ACTION_LIST, ACTION_MTREE, ACTION_COPY_FROM, ACTION_COPY_TO, ACTION_MAKE_ARCHIVE, ACTION_SHIFT));
+        assert(IN_SET(arg_action, ACTION_LIST, ACTION_MTREE, ACTION_MANIFEST, ACTION_COPY_FROM, ACTION_COPY_TO, ACTION_MAKE_ARCHIVE, ACTION_SHIFT));
 
         /* Determine whether to copy ownership:
          * --copy-ownership=yes: always try to preserve ownership
@@ -1535,7 +1803,8 @@ static int action_list_or_mtree_or_copy_or_make_archive(DissectedImage *m, LoopD
         }
 
         case ACTION_LIST:
-        case ACTION_MTREE: {
+        case ACTION_MTREE:
+        case ACTION_MANIFEST: {
                 _cleanup_close_ int dfd = -EBADF;
 
                 dfd = open(root, O_DIRECTORY|O_CLOEXEC|O_RDONLY);
@@ -1548,7 +1817,18 @@ static int action_list_or_mtree_or_copy_or_make_archive(DissectedImage *m, LoopD
                         r = recurse_dir(dfd, NULL, 0, UINT_MAX, RECURSE_DIR_SORT, list_print_item, NULL);
                 else if (arg_action == ACTION_MTREE)
                         r = recurse_dir(dfd, ".", STATX_TYPE|STATX_MODE|STATX_UID|STATX_GID|STATX_SIZE, UINT_MAX, RECURSE_DIR_SORT|RECURSE_DIR_INODE_FD|RECURSE_DIR_TOPLEVEL, mtree_print_item, NULL);
-                else
+                else if (arg_action == ACTION_MANIFEST) {
+                        _cleanup_(hardlink_data_done) HardlinkData hd = {
+                                .db_fd = -EBADF,
+                        };
+                        r = recurse_dir(dfd,
+                                        /* path= */ NULL,
+                                        STATX_TYPE|STATX_MODE|STATX_UID|STATX_GID|STATX_SIZE|STATX_NLINK,
+                                        /* n_depth_max= */ UINT_MAX,
+                                        RECURSE_DIR_SORT|RECURSE_DIR_INODE_FD|RECURSE_DIR_TOPLEVEL,
+                                        manifest_print_item,
+                                        &hd);
+                } else
                         assert_not_reached();
                 if (r < 0)
                         return log_error_errno(r, "Failed to list image: %m");
@@ -2145,11 +2425,12 @@ static int run(int argc, char *argv[]) {
 
         case ACTION_LIST:
         case ACTION_MTREE:
+        case ACTION_MANIFEST:
         case ACTION_COPY_FROM:
         case ACTION_COPY_TO:
         case ACTION_MAKE_ARCHIVE:
         case ACTION_SHIFT:
-                return action_list_or_mtree_or_copy_or_make_archive(m, d, userns_fd);
+                return action_list_or_mtree_or_manifest_or_copy_or_make_archive(m, d, userns_fd);
 
         case ACTION_WITH:
                 return action_with(m, d);