Add libfuse 3 support with libfuse 2 fallback

README.md

@@ -38,7 +38,7 @@ These instructions will get you a copy of the project up and running on your loc
   * automake 1.13.4 or later
   * autoconf 2.69 or later
   * libtool 2.4.2 or later
-  * fuse 2.6.0 or later
+  * fuse3 3.4.0 or later (default), or fuse 2.6.0 or later with `--with-fuse2`
   * uuid 1.36 or later (Linux)
   * libxml-2.0 2.6.16 or later
   * net-snmp 5.3 or later
@@ -198,8 +198,28 @@ make install
 
 `./configure --help` shows various options for build and install.
 
+On Linux the build uses libfuse 3 (package `libfuse3-dev` on Debian/Ubuntu,
+`fuse3-devel` on Fedora/RHEL). Pass `--with-fuse2` to build against the
+legacy libfuse 2 API instead; macOS, FreeBSD, and NetBSD currently use the
+libfuse 2 API by default. On macOS with macFUSE 5 or later, which ships a
+libfuse 3, the FUSE 3 build can be selected with `--with-fuse2=no`
+(autotools) or `-DLTFS_WITH_FUSE2=OFF` (CMake).
+
+FUSE 3 builds negotiate request sizes up to 1 MiB (tunable with
+`-o max_write=<bytes>`), serve directory listings through readdirplus, and
+support `-o direct_io` to bypass the kernel page cache entirely so large
+archive jobs do not fill it (mmap does not work on files opened this way).
+
 In some systems, you might need `sudo ldconfig -v` after `make install` to load the shared libraries correctly.
 
+## Running the test suite
+
+`make check` runs integration tests against a tape emulated in a local
+directory by the file backend; no tape hardware is required. The tests need
+a Linux host with `/dev/fuse` and are skipped elsewhere. On macOS,
+`tests/run-in-docker.sh [configure-options...]` builds and runs the suite
+inside an Ubuntu container.
+
 #### Parameter settings of the sg driver
 
 LTFS uses the sg driver by default. You can improve reliability to change parameters of the sg driver below.

configure.ac

@@ -286,7 +286,55 @@ fi
 dnl
 dnl Check for FUSE, libuuid, and libxml2
 dnl
-PKG_CHECK_MODULES([FUSE_MODULE], [fuse >= 2.6.0])
+dnl libfuse 3 is preferred on Linux, falling back to libfuse 2 when the
+dnl fuse3 development files are not installed. --with-fuse2 forces the
+dnl legacy libfuse 2 API; it is also the default on the other platforms
+dnl until their fuse3 stacks are verified.
+dnl
+AC_ARG_WITH([fuse2],
+    [AS_HELP_STRING([--with-fuse2],
+        [build against libfuse 2 instead of libfuse 3])],
+    [with_fuse2=$withval],
+    [with_fuse2=default])
+if test "x${with_fuse2}" = "xdefault" && test "x${host_linux}" != "xyes"
+then
+    with_fuse2=yes
+fi
+
+if test "x${with_fuse2}" = "xyes"
+then
+    PKG_CHECK_MODULES([FUSE_MODULE], [fuse >= 2.6.0])
+elif test "x${with_fuse2}" = "xno"
+then
+    PKG_CHECK_MODULES([FUSE_MODULE], [fuse3 >= 3.4.0])
+else
+    PKG_CHECK_MODULES([FUSE_MODULE], [fuse3 >= 3.4.0],
+        [with_fuse2=no],
+        [PKG_CHECK_MODULES([FUSE_MODULE], [fuse >= 2.6.0], [with_fuse2=yes])])
+fi
+AC_MSG_CHECKING([for the FUSE API version to use])
+if test "x${with_fuse2}" = "xyes"
+then
+    AC_MSG_RESULT([2])
+else
+    AC_MSG_RESULT([3])
+fi
+
+dnl
+dnl struct fuse_file_info.parallel_direct_writes appeared in libfuse 3.15;
+dnl detect the member instead of relying on version numbers.
+dnl
+if test "x${with_fuse2}" != "xyes"
+then
+    SAVE_CFLAGS=$CFLAGS
+    CFLAGS="$CFLAGS ${FUSE_MODULE_CFLAGS} -DFUSE_USE_VERSION=31"
+    AC_CHECK_MEMBER([struct fuse_file_info.parallel_direct_writes],
+        [AC_DEFINE([HAVE_FUSE_PARALLEL_DIRECT_WRITES], [1],
+            [Define to 1 if struct fuse_file_info has parallel_direct_writes])],
+        [],
+        [[#include <fuse.h>]])
+    CFLAGS="$SAVE_CFLAGS"
+fi
 PKG_CHECK_MODULES([LIBXML2_MODULE], [libxml-2.0 >= 2.6.16])
 
 if test "x${host_mac}" = "xyes"
@@ -458,6 +506,10 @@ dnl Update flags
 dnl Sets CFLAGS to force optimization and debugging options, which isn't quite kosher
 dnl
 AM_CPPFLAGS="-D_GNU_SOURCE -I\$(top_srcdir)/src -DLTFS_CONFIG_FILE='\"${sysconfdir}/ltfs.conf\"' -DLTFS_BASE_DIR='\"${prefix}\"'"
+if test "x${with_fuse2}" != "xyes"
+then
+    AM_CPPFLAGS="${AM_CPPFLAGS} -DHAVE_FUSE3"
+fi
 AM_CFLAGS="-Wall -Wsign-compare -fsigned-char ${FUSE_MODULE_CFLAGS} ${UUID_MODULE_CFLAGS} ${LIBXML2_MODULE_CFLAGS} ${ICU_MODULE_CFLAGS} ${SNMP_ENABLE} ${SNMP_MODULE_CFLAGS}"
 
 if test "x$use_fast" = "xyes"

messages/bin_ltfs/root.txt

@@ -154,7 +154,9 @@ root:table {
 		14115E:string { "Invalid scsi_append_only_mode option: %s." }
                 14116E:string { "This medium is not supported (%d)." }
 		14123W:string { "The main function of FUSE returned error (%d)." }
-
+		14124I:string { "FUSE maximum request size is %u KiB." }
+		14125W:string { "The max_write option is ignored when built against FUSE 2." }
+
 		// 14150 - 14199 are reserved for LE+
 
 		// Help messages
@@ -244,5 +246,7 @@ root:table {
 		// Reserved 14466I
 		14467I:string { "    -o syslogtrace            Enable diagnostic output to stderr and syslog(same as verbose=303)" }
 		// Reserved 14468I
+		14469I:string { "    -o max_write=<num>        Maximum size of a FUSE request in bytes (FUSE 3 builds only, default: 1048576)" }
+		14470I:string { "    -o direct_io              Bypass the kernel page cache for all file I/O (disables mmap)" }
 	}
 }

src/iosched/unified.c

@@ -1790,7 +1790,11 @@ ssize_t _unified_insert_new_request(const char *buf, off_t offset, size_t count,
 	if (new_req->offset + new_req->count > dpr->file_size)
 		dpr->file_size = new_req->offset + new_req->count;
 
-	return (ssize_t)count;
+	/* Only copy_count bytes were stored (one cache block at most); the
+	 * caller's append loop must advance by that, not the full count, or
+	 * everything past the first block of a larger-than-blocksize write is
+	 * silently dropped. */
+	return (ssize_t)copy_count;
 }
 
 /**

src/libltfs/ltfs.h

@@ -210,6 +210,11 @@ struct device_data;
  * or a negative value on error. */
 typedef int (*ltfs_dir_filler) (void *buf, const char *name, void *priv);
 
+/* Directory listing callback that also receives the entry's attributes.
+ * attr may be NULL when the backing store yields names only. */
+typedef int (*ltfs_dir_filler_attr) (void *buf, const char *name,
+	const struct dentry_attr *attr, void *priv);
+
 /**
  * All capacities are relative to filesystem block size.
  */

src/libltfs/ltfs_fsops.c

@@ -1414,6 +1414,96 @@ int ltfs_fsops_readdir(struct dentry *d, void *buf, ltfs_dir_filler filler, void
 	return ret;
 }
 
+/* Copy a child's attributes without taking the volume lock again;
+ * the caller already holds it (same fields as ltfs_fsops_getattr). */
+static void _fsops_child_attr(struct dentry *child, struct dentry_attr *attr,
+	struct ltfs_volume *vol)
+{
+	acquireread_mrsw(&child->meta_lock);
+
+	if (child->isslink)
+		attr->size = strlen(child->target.name);
+	else
+		attr->size = child->size;
+
+	attr->alloc_size = child->realsize;
+	attr->blocksize = vol->label->blocksize;
+	attr->uid = child->uid;
+	attr->nlink = child->link_count;
+	attr->create_time = child->creation_time;
+	attr->access_time = child->access_time;
+	attr->modify_time = child->modify_time;
+	attr->change_time = child->change_time;
+	attr->backup_time = child->backup_time;
+	attr->readonly = child->readonly;
+	attr->isdir = child->isdir;
+	attr->isslink = child->isslink;
+
+	releaseread_mrsw(&child->meta_lock);
+
+	if (! child->isdir && ! child->isslink && iosched_initialized(vol))
+		attr->size = iosched_get_filesize(child, vol);
+}
+
+int ltfs_fsops_readdir_attr(struct dentry *d, void *buf, ltfs_dir_filler_attr filler,
+	void *filler_priv, struct ltfs_volume *vol)
+{
+	int ret = 0;
+	struct name_list *entry, *tmp;
+	struct dentry_attr attr;
+
+	CHECK_ARG_NULL(d, -LTFS_NULL_ARG);
+	CHECK_ARG_NULL(filler, -LTFS_NULL_ARG);
+	CHECK_ARG_NULL(vol, -LTFS_NULL_ARG);
+
+	if (! d->isdir)
+		return -LTFS_ISFILE;
+
+	ret = ltfs_get_volume_lock(false, vol);
+	if (ret < 0)
+		return ret;
+
+	acquireread_mrsw(&d->contents_lock);
+	if (dcache_initialized(vol)) {
+		/* The dentry cache yields names only */
+		int i;
+		char **namelist = NULL;
+		ret = dcache_readdir(d, false, (void ***) &namelist, vol);
+		if (ret == 0 && namelist) {
+			for (i=0; namelist[i]; ++i) {
+				ret = filler(buf, namelist[i], NULL, filler_priv);
+				if (ret < 0)
+					break;
+			}
+			for (i=0; namelist[i]; ++i)
+				free(namelist[i]);
+			free(namelist);
+		}
+	} else {
+		if (HASH_COUNT(d->child_list) != 0) {
+			HASH_SORT(d->child_list, fs_hash_sort_by_uid);
+			HASH_ITER(hh, d->child_list, entry, tmp) {
+				_fsops_child_attr(entry->d, &attr, vol);
+				ret = filler(buf, entry->d->platform_safe_name, &attr, filler_priv);
+				if (ret < 0)
+					break;
+			}
+		}
+	}
+	releaseread_mrsw(&d->contents_lock);
+
+	/* Update access time */
+	if (ret == 0) {
+		acquirewrite_mrsw(&d->meta_lock);
+		get_current_timespec(&d->access_time);
+		releasewrite_mrsw(&d->meta_lock);
+		ltfs_set_index_dirty(true, true, vol->index);
+	}
+
+	releaseread_mrsw(&vol->lock);
+	return ret;
+}
+
 int _ltfs_fsops_read_direntry(struct dentry *d, struct ltfs_direntry *dirent,
 							  unsigned long index, bool root, struct ltfs_volume *vol)
 {

src/libltfs/ltfs_fsops.h

@@ -324,6 +324,13 @@ int ltfs_fsops_removexattr(const char *path, const char *name, ltfs_file_id *id,
 int ltfs_fsops_readdir(struct dentry *d, void *buf, ltfs_dir_filler filler, void *filler_priv,
 	struct ltfs_volume *vol);
 
+/**
+ * List a directory like ltfs_fsops_readdir, passing each entry's attributes
+ * to the filler as well. attr is NULL when the backing store yields names only.
+ */
+int ltfs_fsops_readdir_attr(struct dentry *d, void *buf, ltfs_dir_filler_attr filler,
+	void *filler_priv, struct ltfs_volume *vol);
+
 /**
  * Get an entry in the directory.
  * It does get the "." and ".." entries only when d is specified non volume root directory.

src/libltfs/ltfs_fuse_version.h

@@ -50,6 +50,18 @@
 #ifndef __ltfs_fuse_version_h__
 #define __ltfs_fuse_version_h__
 
+/* HAVE_FUSE3 is set on the compiler command line by configure
+ * (default on Linux; --with-fuse2 selects the libfuse 2 API). */
+#ifdef HAVE_FUSE3
+#define FUSE_USE_VERSION 31
+/* macFUSE 5's libfuse3 defaults to Darwin-specific operation signatures
+ * (struct fuse_darwin_attr, struct statfs, ...). Request the upstream-
+ * compatible API instead; the library exports both symbol flavors. */
+#ifdef __APPLE__
+#define FUSE_DARWIN_ENABLE_EXTENSIONS 0
+#endif
+#else
 #define FUSE_USE_VERSION 26
+#endif
 
 #endif /* __ltfs_fuse_version_h__ */

src/libltfs/xattr.h

@@ -66,7 +66,9 @@ extern "C" {
 #include "libltfs/arch/freebsd/xattr.h"
 #endif
 
-#include "fuse.h"
+#include "libltfs/ltfs_fuse_version.h"
+#include <fuse.h>
+
 #include "ltfs.h"
 
 #define LTFS_PRIVATE_PREFIX "ltfs."