f2fs-tools:sload.f2fs compression support

commit b585244e72
category: bugfix
issue: #I6VAS0
CVE: NA

Signed-off-by: DongSenhao <dongsenhao2@huawei.com>
---------------------------------------

Add F2FS compression support for sload
* Support file extension filter, either default-accept or default-deny
  policy
* Support choice of compression algorithm, LZO (version 2) or LZ4
  (default)
* Support custom log of cluster size
* Support minimum number of compressed blocks per cluster (default 1).
  A cluster will not be compressed if the number can not be met.
* suuport -r (read-only) option
  This releases compressed blocks to secure free space in advance. Note that,
  all compressed files will have the immutable bit.
* Added manpage update
* Remove unecessary qbuf allocation (Jaegeuk, suggested by Satya)

Signed-off-by: Robin Hsu <robinhsu@google.com>
[Jaegeuk Kim: fix some bugs and refactor names]
Signed-off-by: Jaegeuk Kim <jaegeuk@kernel.org>
Signed-off-by: dongsenhao <dongsenhao2@huawei.com>

 create mode 100644 fsck/compress.c
 create mode 100644 fsck/compress.h

Signed-off-by: dongsenhao <dongsenhao2@huawei.com>

configure.ac

@@ -52,6 +52,18 @@ AC_PATH_PROG([LDCONFIG], [ldconfig],
        [$PATH:/sbin])
 
 # Checks for libraries.
+AC_CHECK_LIB([lzo2], [main],
+	[AC_SUBST([liblzo2_LIBS], ["-llzo2"])
+		AC_DEFINE([HAVE_LIBLZO2], [1],
+		[Define if you have liblzo2])
+	], [], [])
+
+AC_CHECK_LIB([lz4], [main],
+	[AC_SUBST([liblz4_LIBS], ["-llz4"])
+		AC_DEFINE([HAVE_LIBLZ4], [1],
+		[Define if you have liblz4])
+	], [], [])
+
 PKG_CHECK_MODULES([libuuid], [uuid])
 
 AS_IF([test "x$with_selinux" != "xno"],

fsck/Makefile.am

@@ -3,12 +3,15 @@
 AM_CPPFLAGS = ${libuuid_CFLAGS} -I$(top_srcdir)/include
 AM_CFLAGS = -Wall
 sbin_PROGRAMS = fsck.f2fs
-noinst_HEADERS = common.h dict.h dqblk_v2.h f2fs.h fsck.h node.h quotaio.h quotaio_tree.h quotaio_v2.h xattr.h
+noinst_HEADERS = common.h dict.h dqblk_v2.h f2fs.h fsck.h node.h quotaio.h \
+		quotaio_tree.h quotaio_v2.h xattr.h compress.h
 include_HEADERS = $(top_srcdir)/include/quota.h
 fsck_f2fs_SOURCES = main.c fsck.c dump.c mount.c defrag.c resize.c \
-		node.c segment.c dir.c sload.c xattr.c \
+		node.c segment.c dir.c sload.c xattr.c compress.c \
 		dict.c mkquota.c quotaio.c quotaio_tree.c quotaio_v2.c
-fsck_f2fs_LDADD = ${libselinux_LIBS} ${libuuid_LIBS} $(top_builddir)/lib/libf2fs.la
+fsck_f2fs_LDADD = ${libselinux_LIBS} ${libuuid_LIBS} \
+	${liblzo2_LIBS} ${liblz4_LIBS} \
+	$(top_builddir)/lib/libf2fs.la
 
 install-data-hook:
 	ln -sf fsck.f2fs $(DESTDIR)/$(sbindir)/dump.f2fs

fsck/compress.c

@@ -0,0 +1,178 @@
+/**
+ * compress.c
+ *
+ * Copyright (c) 2020 Google Inc.
+ *   Robin Hsu <robinhsu@google.com>
+ *  : add sload compression support
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+/* for config.h for general environment (non-Android) */
+#include "f2fs.h"
+
+#include "compress.h"
+#ifdef HAVE_LIBLZO2
+#include <lzo/lzo1x.h>	/* for lzo1x_1_15_compress() */
+#endif
+#ifdef HAVE_LIBLZ4
+#include <lz4.h>	/* for LZ4_compress_fast_extState() */
+#endif
+
+/*
+ * macro/constants borrowed from kernel header (GPL-2.0):
+ * include/linux/lzo.h, and include/linux/lz4.h
+ */
+#ifdef HAVE_LIBLZO2
+#define lzo1x_worst_compress(x)		((x) + (x) / 16 + 64 + 3 + 2)
+#define LZO_WORK_SIZE			ALIGN_UP(LZO1X_1_15_MEM_COMPRESS, 8)
+#endif
+#ifdef HAVE_LIBLZ4
+#define LZ4_MEMORY_USAGE		14
+#define LZ4_MAX_INPUT_SIZE		0x7E000000 /* 2 113 929 216 bytes */
+#ifndef LZ4_STREAMSIZE
+#define LZ4_STREAMSIZE			(LZ4_STREAMSIZE_U64 * sizeof(long long))
+#endif
+#define LZ4_MEM_COMPRESS		LZ4_STREAMSIZE
+#define LZ4_ACCELERATION_DEFAULT	1
+#define LZ4_WORK_SIZE			ALIGN_UP(LZ4_MEM_COMPRESS, 8)
+#endif
+
+#if defined(HAVE_LIBLZO2) || defined(HAVE_LIBLZ4)
+static void reset_cc(struct compress_ctx *cc)
+{
+	memset(cc->rbuf, 0, cc->cluster_size * F2FS_BLKSIZE);
+	memset(cc->cbuf->cdata, 0, cc->cluster_size * F2FS_BLKSIZE
+			- F2FS_BLKSIZE);
+}
+#endif
+
+#ifdef HAVE_LIBLZO2
+static void lzo_compress_init(struct compress_ctx *cc)
+{
+	size_t size = cc->cluster_size * F2FS_BLKSIZE;
+	size_t alloc = size + lzo1x_worst_compress(size)
+			+ COMPRESS_HEADER_SIZE + LZO_WORK_SIZE;
+	cc->private = malloc(alloc);
+	ASSERT(cc->private);
+	cc->rbuf = (char *) cc->private + LZO_WORK_SIZE;
+	cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size);
+}
+
+static int lzo_compress(struct compress_ctx *cc)
+{
+	int ret = lzo1x_1_15_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata,
+			(lzo_uintp)(&cc->clen), cc->private);
+	cc->cbuf->clen = cpu_to_le32(cc->clen);
+	return ret;
+}
+#endif
+
+#ifdef HAVE_LIBLZ4
+static void lz4_compress_init(struct compress_ctx *cc)
+{
+	size_t size = cc->cluster_size * F2FS_BLKSIZE;
+	size_t alloc = size + LZ4_COMPRESSBOUND(size)
+			+ COMPRESS_HEADER_SIZE + LZ4_WORK_SIZE;
+	cc->private = malloc(alloc);
+	ASSERT(cc->private);
+	cc->rbuf = (char *) cc->private + LZ4_WORK_SIZE;
+	cc->cbuf = (struct compress_data *)((char *) cc->rbuf + size);
+}
+
+static int lz4_compress(struct compress_ctx *cc)
+{
+	cc->clen = LZ4_compress_fast_extState(cc->private, cc->rbuf,
+			(char *)cc->cbuf->cdata, cc->rlen,
+			cc->rlen - F2FS_BLKSIZE * c.compress.min_blocks,
+			LZ4_ACCELERATION_DEFAULT);
+
+	if (!cc->clen)
+		return 1;
+
+	cc->cbuf->clen = cpu_to_le32(cc->clen);
+	return 0;
+}
+#endif
+
+const char *supported_comp_names[] = {
+	"lzo",
+	"lz4",
+	"",
+};
+
+compress_ops supported_comp_ops[] = {
+#ifdef HAVE_LIBLZO2
+	{lzo_compress_init, lzo_compress, reset_cc},
+#else
+	{NULL, NULL, NULL},
+#endif
+#ifdef HAVE_LIBLZ4
+	{lz4_compress_init, lz4_compress, reset_cc},
+#else
+	{NULL, NULL, NULL},
+#endif
+};
+
+/* linked list */
+typedef struct _ext_t {
+	const char *ext;
+	struct _ext_t *next;
+} ext_t;
+
+static ext_t *extension_list;
+
+static bool ext_found(const char *ext)
+{
+	ext_t *p = extension_list;
+
+	while (p != NULL && strcmp(ext, p->ext))
+		p = p->next;
+	return (p != NULL);
+}
+
+static const char *get_ext(const char *path)
+{
+	char *p = strrchr(path, '.');
+
+	return p == NULL ? path + strlen(path) : p + 1;
+}
+
+static bool ext_do_filter(const char *path)
+{
+	return (ext_found(get_ext(path)) == true) ^
+		(c.compress.filter == COMPR_FILTER_ALLOW);
+}
+
+static void ext_filter_add(const char *ext)
+{
+	ext_t *node;
+
+	ASSERT(ext != NULL);
+	if (ext_found(ext))
+		return; /* ext was already registered */
+	node = malloc(sizeof(ext_t));
+	ASSERT(node != NULL);
+	node->ext = ext;
+	node->next = extension_list;
+	extension_list = node;
+}
+
+static void ext_filter_destroy(void)
+{
+	ext_t *p;
+
+	while (extension_list != NULL) {
+		p = extension_list;
+		extension_list = p->next;
+		free(p);
+	}
+}
+
+filter_ops ext_filter = {
+	.add = ext_filter_add,
+	.destroy = ext_filter_destroy,
+	.filter = ext_do_filter,
+};

fsck/compress.h

@@ -0,0 +1,22 @@
+/**
+ * compress.h
+ *
+ * Copyright (c) 2020 Google Inc.
+ *   Robin Hsu <robinhsu@google.com>
+ *  : add sload compression support
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#ifndef COMPRESS_H
+#define COMPRESS_H
+
+#include "f2fs_fs.h"
+
+extern const char *supported_comp_names[];
+extern compress_ops supported_comp_ops[];
+extern filter_ops ext_filter;
+
+#endif /* COMPRESS_H */

fsck/fsck.h

@@ -282,7 +282,16 @@ block_t new_node_block(struct f2fs_sb_info *,
 struct quota_file;
 u64 f2fs_quota_size(struct quota_file *);
 u64 f2fs_read(struct f2fs_sb_info *, nid_t, u8 *, u64, pgoff_t);
+enum wr_addr_type {
+	WR_NORMAL = 1,
+	WR_COMPRESS_DATA = 2,
+	WR_NULL_ADDR = NULL_ADDR,		/* 0 */
+	WR_NEW_ADDR = NEW_ADDR,			/* -1U */
+	WR_COMPRESS_ADDR = COMPRESS_ADDR,	/* -2U */
+};
 u64 f2fs_write(struct f2fs_sb_info *, nid_t, u8 *, u64, pgoff_t);
+u64 f2fs_write_compress_data(struct f2fs_sb_info *, nid_t, u8 *, u64, pgoff_t);
+u64 f2fs_write_addrtag(struct f2fs_sb_info *, nid_t, pgoff_t, unsigned int);
 void f2fs_filesize_update(struct f2fs_sb_info *, nid_t, u64);
 
 int get_dnode_of_data(struct f2fs_sb_info *, struct dnode_of_data *,

fsck/main.c

@@ -13,6 +13,9 @@
  * Copyright (c) 2019 Google Inc.
  *   Robin Hsu <robinhsu@google.com>
  *  : add cache layer
+ * Copyright (c) 2020 Google Inc.
+ *   Robin Hsu <robinhsu@google.com>
+ *  : add sload compression support
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -25,6 +28,7 @@
 #include <getopt.h>
 #include <stdbool.h>
 #include "quotaio.h"
+#include "compress.h"
 
 struct f2fs_fsck gfsck;
 
@@ -134,6 +138,17 @@ void sload_usage()
 	MSG(0, "  -S sparse_mode\n");
 	MSG(0, "  -t mount point [prefix of target fs path, default:/]\n");
 	MSG(0, "  -T timestamp\n");
+	MSG(0, "  -c enable compression (default allow policy)\n");
+	MSG(0, "    ------------ Compression sub-options -----------------\n");
+	MSG(0, "    -L <log-of-blocks-per-cluster>, default 2\n");
+	MSG(0, "    -a <algorithm> compression algorithm, default LZ4\n");
+	MSG(0, "    -x <ext> compress files except for these extensions.\n");
+	MSG(0, "    -i <ext> compress files with these extensions only.\n");
+	MSG(0, "    * -i or -x: use it many times for multiple extensions.\n");
+	MSG(0, "    * -i and -x cannot be used together..\n");
+	MSG(0, "    -m <num> min compressed blocks per cluster\n");
+	MSG(0, "    -r readonly (IMMUTABLE) for compressed files\n");
+	MSG(0, "    ------------------------------------------------------\n");
 	MSG(0, "  -d debug level [default:0]\n");
 	MSG(0, "  -V print the version number and exit\n");
 	exit(1);
@@ -534,7 +549,7 @@ void f2fs_parse_options(int argc, char *argv[])
 #endif
 	} else if (!strcmp("sload.f2fs", prog)) {
 #ifdef WITH_SLOAD
-		const char *option_string = "C:d:f:p:s:St:T:V";
+		const char *option_string = "cL:a:i:x:m:rC:d:f:p:s:St:T:V";
 #ifdef HAVE_LIBSELINUX
 		int max_nr_opt = (int)sizeof(c.seopt_file) /
 			sizeof(c.seopt_file[0]);
@@ -543,8 +558,83 @@ void f2fs_parse_options(int argc, char *argv[])
 		char *p;
 
 		c.func = SLOAD;
+		c.compress.cc.log_cluster_size = 2;
+		c.compress.alg = COMPR_LZ4;
+		c.compress.min_blocks = 1;
+		c.compress.filter_ops = &ext_filter;
 		while ((option = getopt(argc, argv, option_string)) != EOF) {
+			unsigned int i;
+			int val;
+
 			switch (option) {
+			case 'c': /* compression support */
+				c.compress.enabled = true;
+				break;
+			case 'L': /* compression: log of blocks-per-cluster */
+				c.compress.required = true;
+				val = atoi(optarg);
+				if (val < MIN_COMPRESS_LOG_SIZE ||
+						val > MAX_COMPRESS_LOG_SIZE) {
+					MSG(0, "\tError: log of blocks per"
+						" cluster must be in the range"
+						" of %d .. %d.\n",
+						MIN_COMPRESS_LOG_SIZE,
+						MAX_COMPRESS_LOG_SIZE);
+					error_out(prog);
+				}
+				c.compress.cc.log_cluster_size = val;
+				break;
+			case 'a': /* compression: choose algorithm */
+				c.compress.required = true;
+				c.compress.alg = MAX_COMPRESS_ALGS;
+				for (i = 0; i < MAX_COMPRESS_ALGS; i++) {
+					if (!strcmp(supported_comp_names[i],
+								optarg)) {
+						c.compress.alg = i;
+						break;
+					}
+				}
+				if (c.compress.alg == MAX_COMPRESS_ALGS) {
+					MSG(0, "\tError: Unknown compression"
+						" algorithm %s\n", optarg);
+					error_out(prog);
+				}
+				break;
+			case 'i': /* compress only these extensions */
+				c.compress.required = true;
+				if (c.compress.filter == COMPR_FILTER_ALLOW) {
+					MSG(0, "\tError: could not mix option"
+							" -i and -x\n");
+					error_out(prog);
+				}
+				c.compress.filter = COMPR_FILTER_DENY;
+				c.compress.filter_ops->add(optarg);
+				break;
+			case 'x': /* compress except for these extensions */
+				c.compress.required = true;
+				if (c.compress.filter == COMPR_FILTER_DENY) {
+					MSG(0, "\tError: could not mix option"
+							" -i and -x\n");
+					error_out(prog);
+				}
+				c.compress.filter = COMPR_FILTER_ALLOW;
+				c.compress.filter_ops->add(optarg);
+				break;
+			case 'm': /* minimum compressed blocks per cluster */
+				c.compress.required = true;
+				val = atoi(optarg);
+				if (val <= 0) {
+					MSG(0, "\tError: minimum compressed"
+						" blocks per cluster must be"
+						" positive.\n");
+					error_out(prog);
+				}
+				c.compress.min_blocks = val;
+				break;
+			case 'r': /* compress file to set IMMUTABLE */
+				c.compress.required = true;
+				c.compress.readonly = true;
+				break;
 			case 'C':
 				c.fs_config_file = absolute_path(optarg);
 				break;
@@ -602,6 +692,27 @@ void f2fs_parse_options(int argc, char *argv[])
 			if (err != NOERROR)
 				break;
 		}
+		if (c.compress.required && !c.compress.enabled) {
+			MSG(0, "\tError: compression sub-options are used"
+				" without the compression enable (-c) option\n"
+			);
+			error_out(prog);
+		}
+		if (err == NOERROR && c.compress.enabled) {
+			c.compress.cc.cluster_size = 1
+				<< c.compress.cc.log_cluster_size;
+			if (c.compress.filter == COMPR_FILTER_UNASSIGNED)
+				c.compress.filter = COMPR_FILTER_ALLOW;
+			if (c.compress.min_blocks >=
+					c.compress.cc.cluster_size) {
+				MSG(0, "\tError: minimum reduced blocks by"
+					" compression per cluster must be at"
+					" most one less than blocks per"
+					" cluster, i.e. %d\n",
+					c.compress.cc.cluster_size - 1);
+				error_out(prog);
+			}
+		}
 #endif /* WITH_SLOAD */
 	}
 
@@ -812,6 +923,30 @@ static int do_resize(struct f2fs_sb_info *sbi)
 #endif
 
 #ifdef WITH_SLOAD
+static int init_compr(struct f2fs_sb_info *sbi)
+{
+	if (!c.compress.enabled)
+		return 0;
+
+	if (!(sbi->raw_super->feature
+			& cpu_to_le32(F2FS_FEATURE_COMPRESSION))) {
+		MSG(0, "Error: Compression (-c) was requested "
+			"but the file system is not created "
+			"with such feature.\n");
+		return -1;
+	}
+	if (!supported_comp_ops[c.compress.alg].init) {
+		MSG(0, "Error: The selected compression algorithm is not"
+				" supported\n");
+		return -1;
+	}
+	c.compress.ops = supported_comp_ops + c.compress.alg;
+	c.compress.ops->init(&c.compress.cc);
+	c.compress.ops->reset(&c.compress.cc);
+	c.compress.cc.rlen = c.compress.cc.cluster_size * F2FS_BLKSIZE;
+	return 0;
+}
+
 static int do_sload(struct f2fs_sb_info *sbi)
 {
 	if (!c.from_dir) {
@@ -821,6 +956,9 @@ static int do_sload(struct f2fs_sb_info *sbi)
 	if (!c.mount_point)
 		c.mount_point = "/";
 
+	if (init_compr(sbi))
+		return -1;
+
 	return f2fs_sload(sbi);
 }
 #endif
@@ -971,6 +1109,9 @@ retry:
 		return ret2;
 	}
 
+	if (c.func == SLOAD)
+		c.compress.filter_ops->destroy();
+
 	printf("\nDone: %lf secs\n", (get_boottime_ns() - start) / 1000000000.0);
 	return ret;
 

fsck/segment.c

@@ -8,6 +8,9 @@
  *   Hou Pengyang <houpengyang@huawei.com>
  *   Liu Shuoran <liushuoran@huawei.com>
  *   Jaegeuk Kim <jaegeuk@kernel.org>
+ * Copyright (c) 2020 Google Inc.
+ *   Robin Hsu <robinhsu@google.com>
+ *  : add sload compression support
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License version 2 as
@@ -111,6 +114,8 @@ int new_data_block(struct f2fs_sb_info *sbi, void *block,
 
 	get_node_info(sbi, dn->nid, &ni);
 	set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
+
+	dn->data_blkaddr = blkaddr;
 	ret = reserve_new_block(sbi, &dn->data_blkaddr, &sum, type, 0);
 	if (ret) {
 		c.alloc_failed = 1;
@@ -228,8 +233,14 @@ u64 f2fs_read(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
 	return read_count;
 }
 
-u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
+/*
-					u64 count, pgoff_t offset)
+ * Do not call this function directly.  Instead, call one of the following:
+ *     u64 f2fs_write();
+ *     u64 f2fs_write_compress_data();
+ *     u64 f2fs_write_addrtag();
+ */
+static u64 f2fs_write_ex(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
+		u64 count, pgoff_t offset, enum wr_addr_type addr_type)
 {
 	struct dnode_of_data dn;
 	struct node_info ni;
@@ -243,6 +254,19 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
 	void* index_node = NULL;
 	int idirty = 0;
 	int err;
+	bool has_data = (addr_type == WR_NORMAL
+			|| addr_type == WR_COMPRESS_DATA);
+
+	if (count == 0)
+		return 0;
+
+	/*
+	 * Enforce calling from f2fs_write(), f2fs_write_compress_data(),
+	 * and f2fs_write_addrtag().   Beside, check if is properly called.
+	 */
+	ASSERT((!has_data && buffer == NULL) || (has_data && buffer != NULL));
+	if (addr_type != WR_NORMAL)
+		ASSERT(offset % F2FS_BLKSIZE == 0); /* block boundary only */
 
 	/* Memory allocation for block buffer and inode. */
 	blk_buffer = calloc(BLOCK_SZ, 2);
@@ -265,15 +289,26 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
 			if (err)
 				break;
 			idirty |= dn.idirty;
-			if (index_node)
+			free(index_node);
-				free(index_node);
 			index_node = (dn.node_blk == dn.inode_blk) ?
-							NULL : dn.node_blk;
+					NULL : dn.node_blk;
 			remained_blkentries = ADDRS_PER_PAGE(sbi,
-						dn.node_blk, dn.inode_blk);
+					dn.node_blk, dn.inode_blk) -
+					dn.ofs_in_node;
 		}
 		ASSERT(remained_blkentries > 0);
 
+		if (!has_data) {
+			dn.data_blkaddr = addr_type;
+			set_data_blkaddr(&dn);
+			idirty |= dn.idirty;
+			if (dn.ndirty)
+				ASSERT(dev_write_block(dn.node_blk,
+						dn.node_blkaddr) >= 0);
+			written_count = 0;
+			break;
+		}
+
 		blkaddr = datablock_addr(dn.node_blk, dn.ofs_in_node);
 		if (blkaddr == NULL_ADDR || blkaddr == NEW_ADDR) {
 			err = new_data_block(sbi, blk_buffer,
@@ -281,6 +316,7 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
 			if (err)
 				break;
 			blkaddr = dn.data_blkaddr;
+			idirty |= dn.idirty;
 		}
 
 		off_in_blk = offset % BLOCK_SZ;
@@ -305,9 +341,10 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
 
 		dn.ofs_in_node++;
 		if ((--remained_blkentries == 0 || count == 0) && (dn.ndirty))
-			ASSERT(dev_write_block(dn.node_blk, dn.node_blkaddr) >= 0);
+			ASSERT(dev_write_block(dn.node_blk, dn.node_blkaddr)
+					>= 0);
 	}
-	if (offset > le64_to_cpu(inode->i.i_size)) {
+	if (addr_type == WR_NORMAL && offset > le64_to_cpu(inode->i.i_size)) {
 		inode->i.i_size = cpu_to_le64(offset);
 		idirty = 1;
 	}
@@ -315,13 +352,33 @@ u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
 		ASSERT(inode == dn.inode_blk);
 		ASSERT(write_inode(inode, ni.blk_addr) >= 0);
 	}
-	if (index_node)
+
-		free(index_node);
+	free(index_node);
 	free(blk_buffer);
 
 	return written_count;
 }
 
+u64 f2fs_write(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
+					u64 count, pgoff_t offset)
+{
+	return f2fs_write_ex(sbi, ino, buffer, count, offset, WR_NORMAL);
+}
+
+u64 f2fs_write_compress_data(struct f2fs_sb_info *sbi, nid_t ino, u8 *buffer,
+					u64 count, pgoff_t offset)
+{
+	return f2fs_write_ex(sbi, ino, buffer, count, offset, WR_COMPRESS_DATA);
+}
+
+u64 f2fs_write_addrtag(struct f2fs_sb_info *sbi, nid_t ino, pgoff_t offset,
+		unsigned int addrtag)
+{
+	ASSERT(addrtag == COMPRESS_ADDR || addrtag == NEW_ADDR
+			|| addrtag == NULL_ADDR);
+	return f2fs_write_ex(sbi, ino, NULL, F2FS_BLKSIZE, offset, addrtag);
+}
+
 /* This function updates only inode->i.i_size */
 void f2fs_filesize_update(struct f2fs_sb_info *sbi, nid_t ino, u64 filesize)
 {
@@ -342,11 +399,59 @@ void f2fs_filesize_update(struct f2fs_sb_info *sbi, nid_t ino, u64 filesize)
 	free(inode);
 }
 
+#define MAX_BULKR_RETRY 5
+int bulkread(int fd, void *rbuf, size_t rsize, bool *eof)
+{
+	int n = 0;
+	int retry = MAX_BULKR_RETRY;
+	int cur;
+
+	if (!rsize)
+		return 0;
+
+	if (eof != NULL)
+		*eof = false;
+	while (rsize && (cur = read(fd, rbuf, rsize)) != 0) {
+		if (cur == -1) {
+			if (errno == EINTR && retry--)
+				continue;
+			return -1;
+		}
+		retry = MAX_BULKR_RETRY;
+
+		rsize -= cur;
+		n += cur;
+	}
+	if (eof != NULL)
+		*eof = (cur == 0);
+	return n;
+}
+
+u64 f2fs_fix_mutable(struct f2fs_sb_info *sbi, nid_t ino, pgoff_t offset,
+		unsigned int compressed)
+{
+	unsigned int i;
+	u64 wlen;
+
+	if (c.compress.readonly)
+		return 0;
+
+	for (i = 0; i < compressed - 1; i++) {
+		wlen = f2fs_write_addrtag(sbi, ino,
+				offset + (i << F2FS_BLKSIZE_BITS), NEW_ADDR);
+		if (wlen)
+			return wlen;
+	}
+	return 0;
+}
+
 int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de)
 {
 	int fd, n;
 	pgoff_t off = 0;
 	u8 buffer[BLOCK_SZ];
+	struct node_info ni;
+	struct f2fs_node *node_blk;
 
 	if (de->ino == 0)
 		return -1;
@@ -359,8 +464,6 @@ int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de)
 
 	/* inline_data support */
 	if (de->size <= DEF_MAX_INLINE_DATA) {
-		struct node_info ni;
-		struct f2fs_node *node_blk;
 		int ret;
 
 		get_node_info(sbi, de->ino, &ni);
@@ -385,6 +488,86 @@ int f2fs_build_file(struct f2fs_sb_info *sbi, struct dentry *de)
 		node_blk->i.i_size = cpu_to_le64(de->size);
 		ASSERT(write_inode(node_blk, ni.blk_addr) >= 0);
 		free(node_blk);
+#ifdef WITH_SLOAD
+	} else if (c.func == SLOAD && c.compress.enabled &&
+			c.compress.filter_ops->filter(de->full_path)) {
+		bool eof = false;
+		u8 *rbuf = c.compress.cc.rbuf;
+		unsigned int cblocks = 0;
+
+		node_blk = calloc(BLOCK_SZ, 1);
+		ASSERT(node_blk);
+
+		/* read inode */
+		get_node_info(sbi, de->ino, &ni);
+		ASSERT(dev_read_block(node_blk, ni.blk_addr) >= 0);
+		/* update inode meta */
+		node_blk->i.i_compress_algrithm = c.compress.alg;
+		node_blk->i.i_log_cluster_size =
+				c.compress.cc.log_cluster_size;
+		node_blk->i.i_flags = cpu_to_le32(
+				F2FS_COMPR_FL |
+				(c.compress.readonly ? FS_IMMUTABLE_FL : 0));
+		ASSERT(write_inode(node_blk, ni.blk_addr) >= 0);
+
+		while (!eof && (n = bulkread(fd, rbuf, c.compress.cc.rlen,
+				&eof)) > 0) {
+			int ret = c.compress.ops->compress(&c.compress.cc);
+			u64 wlen;
+			u32 csize = ALIGN_UP(c.compress.cc.clen +
+					COMPRESS_HEADER_SIZE, BLOCK_SZ);
+			unsigned int cur_cblk;
+
+			if (ret || n < c.compress.cc.rlen ||
+				n < (int)(csize + BLOCK_SZ *
+						c.compress.min_blocks)) {
+				wlen = f2fs_write(sbi, de->ino, rbuf, n, off);
+				ASSERT((int)wlen == n);
+			} else {
+				wlen = f2fs_write_addrtag(sbi, de->ino, off,
+						WR_COMPRESS_ADDR);
+				ASSERT(!wlen);
+				wlen = f2fs_write_compress_data(sbi, de->ino,
+						(u8 *)c.compress.cc.cbuf,
+						csize, off + BLOCK_SZ);
+				ASSERT(wlen == csize);
+				c.compress.ops->reset(&c.compress.cc);
+				cur_cblk = (c.compress.cc.rlen - csize) /
+								BLOCK_SZ;
+				cblocks += cur_cblk;
+				wlen = f2fs_fix_mutable(sbi, de->ino,
+						off + BLOCK_SZ + csize,
+						cur_cblk);
+				ASSERT(!wlen);
+			}
+			off += n;
+		}
+		if (n == -1) {
+			fprintf(stderr, "Load file '%s' failed: ",
+					de->full_path);
+			perror(NULL);
+		}
+		/* read inode */
+		get_node_info(sbi, de->ino, &ni);
+		ASSERT(dev_read_block(node_blk, ni.blk_addr) >= 0);
+		/* update inode meta */
+		node_blk->i.i_size = cpu_to_le64(off);
+		if (!c.compress.readonly) {
+			node_blk->i.i_compr_blocks = cpu_to_le64(cblocks);
+			node_blk->i.i_blocks += cpu_to_le64(cblocks);
+		}
+		ASSERT(write_inode(node_blk, ni.blk_addr) >= 0);
+		free(node_blk);
+
+		if (!c.compress.readonly) {
+			sbi->total_valid_block_count += cblocks;
+			if (sbi->total_valid_block_count >=
+					sbi->user_block_count) {
+				ERR_MSG("Not enough space\n");
+				ASSERT(0);
+			}
+		}
+#endif
 	} else {
 		while ((n = read(fd, buffer, BLOCK_SZ)) > 0) {
 			f2fs_write(sbi, de->ino, buffer, n, off);

include/f2fs_fs.h

@@ -5,6 +5,9 @@
  *             http://www.samsung.com/
  * Copyright (c) 2019 Google Inc.
  *             http://www.google.com/
+ * Copyright (c) 2020 Google Inc.
+ *   Robin Hsu <robinhsu@google.com>
+ *  : add sload compression support
  *
  * Dual licensed under the GPL or LGPL version 2 licenses.
  *
@@ -68,6 +71,10 @@ typedef uint16_t u_int16_t;
 typedef uint8_t u_int8_t;
 #endif
 
+/* codes from kernel's f2fs.h, GPL-v2.0 */
+#define MIN_COMPRESS_LOG_SIZE	2
+#define MAX_COMPRESS_LOG_SIZE	8
+
 typedef u_int64_t	u64;
 typedef u_int32_t	u32;
 typedef u_int16_t	u16;
@@ -93,6 +100,31 @@ typedef u32	__be32;
 typedef u64	__be64;
 #endif
 
+/*
+ * code borrowed from kernel f2fs dirver: f2fs.h, GPL-2.0
+ *  : definitions of COMPRESS_DATA_RESERVED_SIZE,
+ *    struct compress_data, COMPRESS_HEADER_SIZE,
+ *    and struct compress_ctx
+ */
+#define COMPRESS_DATA_RESERVED_SIZE		4
+struct compress_data {
+	__le32 clen;			/* compressed data size */
+	__le32 chksum;			/* checksum of compressed data */
+	__le32 reserved[COMPRESS_DATA_RESERVED_SIZE];	/* reserved */
+	u8 cdata[];			/* compressed data */
+};
+#define COMPRESS_HEADER_SIZE	(sizeof(struct compress_data))
+/* compress context */
+struct compress_ctx {
+	unsigned int cluster_size;	/* page count in cluster */
+	unsigned int log_cluster_size;	/* log of cluster size */
+	void *rbuf;			/* compression input buffer */
+	struct compress_data *cbuf;	/* comprsssion output header + data */
+	size_t rlen;			/* valid data length in rbuf */
+	size_t clen;			/* valid data length in cbuf */
+	void *private;			/* work buf for compress algorithm */
+};
+
 #if HAVE_BYTESWAP_H
 #include <byteswap.h>
 #else
@@ -345,6 +377,47 @@ typedef struct {
 	bool dbg_en;
 } dev_cache_config_t;
 
+/* f2fs_configration for compression used for sload.f2fs */
+typedef struct  {
+	void (*init)(struct compress_ctx *cc);
+	int (*compress)(struct compress_ctx *cc);
+	void (*reset)(struct compress_ctx *cc);
+} compress_ops;
+
+/* Should be aligned to supported_comp_names and support_comp_ops */
+enum compress_algorithms {
+	COMPR_LZO,
+	COMPR_LZ4,
+	MAX_COMPRESS_ALGS,
+};
+
+enum filter_policy {
+	COMPR_FILTER_UNASSIGNED = 0,
+	COMPR_FILTER_ALLOW,
+	COMPR_FILTER_DENY,
+};
+
+typedef struct {
+	void (*add)(const char *);
+	void (*destroy)(void);
+	bool (*filter)(const char *);
+} filter_ops;
+
+typedef struct {
+	bool enabled;			/* disabled by default */
+	bool required;			/* require to enable */
+	bool readonly;			/* readonly to release blocks */
+	struct compress_ctx cc;		/* work context */
+	enum compress_algorithms alg;	/* algorithm to compress */
+	compress_ops *ops;		/* ops per algorithm */
+	unsigned int min_blocks;	/* save more blocks than this */
+	enum filter_policy filter;	/* filter to try compression */
+	filter_ops *filter_ops;		/* filter ops */
+} compress_config_t;
+
+#define ALIGN_UP(value, size) ((value) + ((value) % (size) > 0 ? \
+		(size) - (value) % (size) : 0))
+
 struct f2fs_configuration {
 	u_int32_t reserved_segments;
 	u_int32_t new_reserved_segments;
@@ -441,6 +514,9 @@ struct f2fs_configuration {
 
 	/* cache parameters */
 	dev_cache_config_t cache_config;
+
+	/* compression support for sload.f2fs */
+	compress_config_t compress;
 };
 
 #ifdef CONFIG_64BIT
@@ -1377,7 +1453,7 @@ int f2fs_reset_zone(int, void *);
 extern int f2fs_reset_zones(int);
 extern uint32_t f2fs_get_usable_segments(struct f2fs_super_block *sb);
 
-#define SIZE_ALIGN(val, size)	((val) + (size) - 1) / (size)
+#define SIZE_ALIGN(val, size)	(((val) + (size) - 1) / (size))
 #define SEG_ALIGN(blks)		SIZE_ALIGN(blks, c.blks_per_seg)
 #define ZONE_ALIGN(blks)	SIZE_ALIGN(blks, c.blks_per_seg * \
 					c.segs_per_zone)

lib/libf2fs_io.c

@@ -5,6 +5,9 @@
  *             http://www.samsung.com/
  * Copyright (c) 2019 Google Inc.
  *             http://www.google.com/
+ * Copyright (c) 2020 Google Inc.
+ *   Robin Hsu <robinhsu@google.com>
+ *  : add quick-buffer for sload compression support
  *
  * Dual licensed under the GPL or LGPL version 2 licenses.
  */

man/sload.f2fs.8

@@ -7,22 +7,48 @@ sload.f2fs \- load directories and files into the device directly
 .B sload.f2fs
 [
 .B \-f
-.I source directory path
+.I source-directory-path
 ]
 [
 .B \-t
-.I mount point
+.I mount-point
 ]
 [
 .B \-d
 .I debugging-level
 ]
+[
+.B \-c
+[
+.B \-L
+.I log-of-blocks-per-cluster
+]
+[
+.B \-a
+.I compression-algorithm
+]
+[
+.B \-x
+.I file-extension-to-exclude-from-compression
+|
+.B \-i
+.I file-extension-to-include-for-compression
+]
+[
+.B \-m
+.I minimum-compressed-blocks-per-cluster
+]
+[
+.B \-r
+]
+]
 .I device
 .SH DESCRIPTION
 .B sload.f2fs
-is used to load directories and files into a disk partition.
+is used to load directories and files into a disk partition, or an F2FS
-\fIdevice\fP is the special file corresponding to the device (e.g.
+image (file).
-\fI/dev/sdXX\fP).
+\fIdevice\fP could a special file corresponding to the device (e.g.
+\fI/dev/sdXX\fP), or an F2FS image file.
 
 .PP
 The exit code returned by
@@ -30,24 +56,72 @@ The exit code returned by
 is 0 on success and -1 on failure.
 .SH OPTIONS
 .TP
-.BI \-f " source directory path"
+.BI \-f " source-directory-path"
 Specify the source directory path to be loaded.
 .TP
-.BI \-t " mount point path"
+.BI \-t " mount-point-path"
 Specify the mount point path in the partition to load.
 .TP
 .BI \-d " debug-level"
 Specify the level of debugging options.
 The default number is 0, which shows basic debugging messages.
 .TP
+.BI \-c
+Enable a cluster-based file compression.
+The file would be chopped into clusters, and each cluster is compressed
+independently.
+.TP
+.BI \-L " log-of-blocks-per-cluster
+Specify cluster size in power of two blocks.
+The minimum value is 2 (4 blocks, default).
+The maximum value is 8 (256 blocks).
+Note that a block contains 4096 bytes.
+This option must be used with option \fB\-c\fR.
+.TP
+.BI \-a " compression-algorithm"
+Choose the algorithm for compression. Available options are:
+lzo, lz4 (default).
+This option must be used with option \fB\-c\fR.
+.TP
+.BI \-i " file-extension-to-include-for-compression"
+Specify a file extension to include for the compression.
+To specify multiple file extensions, use multiple option \fB\-i\fR's.
+Files having one of the listed extensions will be compressed.
+This option must be used with option \fB\-c\fR.
+.TP
+.BI \-x " file-extension-to-exclude-from-compression"
+Specify a file extension to exclude from compression.
+To specify multiple file extensions, use multiple option \fB\-x\fR's.
+Files having one of the listed extensions won't be compressed.
+This option must be used with option \fB\-c\fR.
+.TP
+.BI \-m " minimum-compressed-blocks-per-cluster"
+Specify a minimum block count saved (by compression) per cluster.
+The minimum value is 1 (default).
+Maximum value is the cluster size in blocks minus 1.
+If compression of a cluster fails to save at least the minimum compressed
+block count given by the option, the cluster will not be compressed.
+This option must be used with option \fB\-c\fR.
+.TP
+.BI \-r
+Specify read-only flag for the compressed files.
+It allows filesystem to release compressed space to the users, since, without
+this option, filesystem should keep the space for future file updates.
+This option must be used with option \fB\-c\fR.
+
+.SH NOTES
+If neither \fB\-i\fR nor \fB\-x\fR is used, all files will be compressed.
+Obviously, option \fB\-i\fR and \fB-x\fR can not be used together.
+
 .SH AUTHOR
 This version of
 .B sload.f2fs
-has been written by Hou Pengyang <houpengyang@huawei.com>,
+has been contributed by Hou Pengyang <houpengyang@huawei.com>,
-Liu Shuoran <liushuoran@huawei.com>, Jaegeuk Kim <jaegeuk@kernel.org>
+Liu Shuoran <liushuoran@huawei.com>, Jaegeuk Kim <jaegeuk@kernel.org>,
+Robin Hsu <robinhsu@google.com>
 .SH AVAILABILITY
 .B sload.f2fs
-is available from git://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs-tools.git.
+is available from <git://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs-tools.git>.
 .SH SEE ALSO
 .BR mkfs.f2fs(8),
 .BR fsck.f2fs(8),