diff --git a/cmake/dependencies.cmake b/cmake/dependencies.cmake index 959a8ede..e934f274 100644 --- a/cmake/dependencies.cmake +++ b/cmake/dependencies.cmake @@ -51,6 +51,31 @@ else() import_pkgconfig_target(TARGET_NAME xz PKGCONFIG_TARGET liblzma) endif() +set(USE_SYSTEM_ZSTD OFF CACHE BOOL "Use system zstd instead of building our own") + +if(NOT USE_SYSTEM_ZSTD) + message(STATUS "Downloading and building zstd") + + ExternalProject_Add(zstd-EXTERNAL + URL https://github.com/facebook/zstd/releases/download/v1.5.0/zstd-1.5.0.tar.gz + URL_HASH SHA512=b322fc1b89a556827b7fece2fb0f34c83bf65bb85b2468c791d6d9178a65c81e21f4171b7533cbf12bc1dfb2fd323d3e8c34d86167b157645c27f65186eec659 + CONFIGURE_COMMAND echo configure zstd + BUILD_COMMAND CC=${CC} CXX=${CXX} CFLAGS=-fPIC CPPFLAGS=${CPPFLAGS} LDFLAGS=${LDFLAGS} ${MAKE} -C/lib ZSTD_LIB_MINIFY=1 libzstd.a + INSTALL_COMMAND ${MAKE} -C/lib PREFIX= install-static install-includes + ) + + import_external_project( + TARGET_NAME zstd + EXT_PROJECT_NAME zstd-EXTERNAL + LIBRARY_DIRS /lib/ + LIBRARIES "/lib/libzstd.a" + INCLUDE_DIRS "/lib/" + ) +else() + message(STATUS "Using system zstd") + + import_pkgconfig_target(TARGET_NAME zstd PKGCONFIG_TARGET libzstd) +endif() # as distros don't provide suitable squashfuse and squashfs-tools, those dependencies are bundled in, can, and should # be used from this repository for AppImageKit @@ -83,8 +108,9 @@ if(NOT USE_SYSTEM_SQUASHFUSE) COMMAND ${AUTORECONF} -fi || true COMMAND ${SED} -i "/PKG_CHECK_MODULES.*/,/,:./d" configure # https://github.com/vasi/squashfuse/issues/12 COMMAND ${SED} -i "s/typedef off_t sqfs_off_t/typedef int64_t sqfs_off_t/g" common.h # off_t's size might differ, see https://stackoverflow.com/a/9073762 - COMMAND CC=${CC} CXX=${CXX} CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} /configure --disable-demo --disable-high-level --without-lzo --without-lz4 --prefix= --libdir=/lib --with-xz=${xz_PREFIX} ${EXTRA_CONFIGURE_FLAGS} + COMMAND CC=${CC} CXX=${CXX} CFLAGS=${CFLAGS} LDFLAGS=${LDFLAGS} /configure --disable-demo --disable-high-level --without-lzo --without-lz4 --prefix= --libdir=/lib --with-xz=${xz_PREFIX} --with-zstd=${zstd_PREFIX} ${EXTRA_CONFIGURE_FLAGS} COMMAND ${SED} -i "s|XZ_LIBS = -llzma |XZ_LIBS = -Bstatic ${xz_LIBRARIES}/|g" Makefile + COMMAND ${SED} -i "s|ZSTD_LIBS = |ZSTD_LIBS = -Bstatic ${zstd_LIBRARIES}|g" Makefile BUILD_COMMAND ${MAKE} BUILD_IN_SOURCE ON INSTALL_COMMAND ${MAKE} install @@ -141,6 +167,13 @@ if(TARGET xz-EXTERNAL) endif() endif() +# only have to build custom zstd when not using system libzstd +if(TARGET zstd-EXTERNAL) + if(TARGET squashfuse-EXTERNAL) + ExternalProject_Add_StepDependencies(squashfuse-EXTERNAL configure zstd-EXTERNAL) + endif() +endif() + ## Boost if(NOT USE_SYSTEM_BOOST) message(STATUS "Downloading and building boost") diff --git a/include/appimage/appimage_legacy.h b/include/appimage/appimage_legacy.h index e3895bc8..20351bbb 100644 --- a/include/appimage/appimage_legacy.h +++ b/include/appimage/appimage_legacy.h @@ -10,23 +10,6 @@ * All of the functions in this header are deprecated and must not be used in newly written code */ -/* - * Calculate the size of an ELF file on disk based on the information in its header - * - * Example: - * - * ls -l 126584 - * - * Calculation using the values also reported by readelf -h: - * Start of section headers e_shoff 124728 - * Size of section headers e_shentsize 64 - * Number of section headers e_shnum 29 - * - * e_shoff + ( e_shentsize * e_shnum ) = 126584 - */ -ssize_t appimage_get_elf_size(const char* fname) __attribute__ ((deprecated)); - - /* * Checks whether a type 1 AppImage's desktop file has set Terminal=true. * diff --git a/include/appimage/appimage_shared.h b/include/appimage/appimage_shared.h index 88db4375..eec58b59 100644 --- a/include/appimage/appimage_shared.h +++ b/include/appimage/appimage_shared.h @@ -38,6 +38,22 @@ char* appimage_hexlify(const char* bytes, size_t numBytes); */ bool appimage_type2_digest_md5(const char* fname, char* digest); +/* + * Calculate the size of an ELF file on disk based on the information in its header + * + * Example: + * + * ls -l 126584 + * + * Calculation using the values also reported by readelf -h: + * Start of section headers e_shoff 124728 + * Size of section headers e_shentsize 64 + * Number of section headers e_shnum 29 + * + * e_shoff + ( e_shentsize * e_shnum ) = 126584 + */ +ssize_t appimage_get_elf_size(const char* fname); + #ifdef __cplusplus } #endif diff --git a/src/libappimage/CMakeLists.txt b/src/libappimage/CMakeLists.txt index 6b24f580..1867a05d 100644 --- a/src/libappimage/CMakeLists.txt +++ b/src/libappimage/CMakeLists.txt @@ -33,6 +33,7 @@ foreach(target libappimage libappimage_static) # unit tests etc., which use squashfuse directly, must link to it explicitly PRIVATE libsquashfuse PRIVATE xz + PRIVATE zstd PRIVATE Boost::filesystem PUBLIC libappimage_shared PUBLIC pthread diff --git a/src/libappimage/libappimage_legacy.cpp b/src/libappimage/libappimage_legacy.cpp index 9448934a..f18abd16 100644 --- a/src/libappimage/libappimage_legacy.cpp +++ b/src/libappimage/libappimage_legacy.cpp @@ -2,10 +2,6 @@ #include extern "C" { -ssize_t appimage_get_elf_size(const char* fname) { - return appimage_get_payload_offset(fname); -} - int appimage_type1_is_terminal_app(const char* path) { return appimage_is_terminal_app(path); } ; diff --git a/src/libappimage_shared/elf.c b/src/libappimage_shared/elf.c index c925feec..3144fbd6 100644 --- a/src/libappimage_shared/elf.c +++ b/src/libappimage_shared/elf.c @@ -117,6 +117,41 @@ static off_t read_elf64(FILE* fd) return sht_end > last_section_end ? sht_end : last_section_end; } +ssize_t appimage_get_elf_size(const char* fname) { + off_t ret; + FILE* fd = NULL; + off_t size = -1; + + fd = fopen(fname, "rb"); + if (fd == NULL) { + fprintf(stderr, "Cannot open %s: %s\n", + fname, strerror(errno)); + return -1; + } + ret = fread(ehdr.e_ident, 1, EI_NIDENT, fd); + if (ret != EI_NIDENT) { + fprintf(stderr, "Read of e_ident from %s failed: %s\n", + fname, strerror(errno)); + return -1; + } + if ((ehdr.e_ident[EI_DATA] != ELFDATA2LSB) && + (ehdr.e_ident[EI_DATA] != ELFDATA2MSB)) { + fprintf(stderr, "Unkown ELF data order %u\n", + ehdr.e_ident[EI_DATA]); + return -1; + } + if (ehdr.e_ident[EI_CLASS] == ELFCLASS32) { + size = read_elf32(fd); + } else if (ehdr.e_ident[EI_CLASS] == ELFCLASS64) { + size = read_elf64(fd); + } else { + fprintf(stderr, "Unknown ELF class %u\n", ehdr.e_ident[EI_CLASS]); + return -1; + } + + fclose(fd); + return size; +} /* Return the offset, and the length of an ELF section with a given name in a given ELF file */ bool appimage_get_elf_section_offset_and_length(const char* fname, const char* section_name, unsigned long* offset, unsigned long* length) { diff --git a/src/patches/patch-squashfuse.sh.in b/src/patches/patch-squashfuse.sh.in index bd3452f2..a552c654 100755 --- a/src/patches/patch-squashfuse.sh.in +++ b/src/patches/patch-squashfuse.sh.in @@ -4,5 +4,6 @@ git checkout ll.c Makefile.am fuseprivate.c fuseprivate.h hl.c ll.h ll_inode.c n patch -p1 < @PROJECT_SOURCE_DIR@/src/patches/squashfuse.patch patch -p1 < @PROJECT_SOURCE_DIR@/src/patches/squashfuse_dlopen.patch +patch -p1 < @PROJECT_SOURCE_DIR@/src/patches/squashfuse_zstd_support.patch cp -v @PROJECT_SOURCE_DIR@/src/patches/squashfuse_dlopen.c @PROJECT_SOURCE_DIR@/src/patches/squashfuse_dlopen.h . diff --git a/src/patches/squashfuse_zstd_support.patch b/src/patches/squashfuse_zstd_support.patch new file mode 100644 index 00000000..f9d41be8 --- /dev/null +++ b/src/patches/squashfuse_zstd_support.patch @@ -0,0 +1,112 @@ +From 5986f1d2b002d4f7f7144239a2c39b8aa39ac874 Mon Sep 17 00:00:00 2001 +From: Sean Purcell +Date: Mon, 27 Mar 2017 14:42:40 -0700 +Subject: [PATCH] Add zstd compression support + +--- + CONFIGURATION | 1 + + Makefile.am | 4 ++-- + configure.ac | 1 + + decompress.c | 21 ++++++++++++++++++++- + squashfs_fs.h | 1 + + 5 files changed, 25 insertions(+), 3 deletions(-) + +diff --git a/CONFIGURATION b/CONFIGURATION +index c172c3e..044c67a 100644 +--- a/CONFIGURATION ++++ b/CONFIGURATION +@@ -14,5 +14,6 @@ These are the most useful options to ./configure: + --with-xz=PREFIX + --with-lzo=PREFIX + --with-lz4=PREFIX ++ --with-zstd=PREFIX + + More options are available in `./configure --help' +diff --git a/Makefile.am b/Makefile.am +index e84534e..8e61e21 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -1,4 +1,4 @@ +-COMPRESSION_LIBS = $(ZLIB_LIBS) $(XZ_LIBS) $(LZO_LIBS) $(LZ4_LIBS) ++COMPRESSION_LIBS = $(ZLIB_LIBS) $(XZ_LIBS) $(LZO_LIBS) $(LZ4_LIBS) $(ZSTD_LIBS) + + ACLOCAL_AMFLAGS = -I m4 --install + +@@ -23,2 +23,2 @@ + libsquashfuse_la_CPPFLAGS = $(ZLIB_CPPFLAGS) $(XZ_CPPFLAGS) $(LZO_CPPFLAGS) \ +- $(LZ4_CPPFLAGS) ++ $(LZ4_CPPFLAGS) $(ZSTD_CPPFLAGS) +diff --git a/configure.ac b/configure.ac +index 36ffb51..f3b55eb 100644 +--- a/configure.ac ++++ b/configure.ac +@@ -31,6 +31,7 @@ SQ_CHECK_DECOMPRESS([ZLIB],[z],[uncompress],[zlib.h]) + SQ_CHECK_DECOMPRESS([XZ],[lzma],[lzma_stream_buffer_decode],[lzma.h],[liblzma]) + SQ_CHECK_DECOMPRESS([LZO],[lzo2],[lzo1x_decompress_safe],[lzo/lzo1x.h]) + SQ_CHECK_DECOMPRESS([LZ4],[lz4],[LZ4_decompress_safe],[lz4.h]) ++SQ_CHECK_DECOMPRESS([ZSTD],[zstd],[ZSTD_decompress],[zstd.h]) + AS_IF([test "x$sq_decompressors" = x], + [AC_MSG_FAILURE([At least one decompression library must exist])]) + +diff --git a/decompress.c b/decompress.c +index d8a677e..80344f0 100644 +--- a/decompress.c ++++ b/decompress.c +@@ -98,6 +98,19 @@ static sqfs_err sqfs_decompressor_lz4(void *in, size_t insz, + #endif + + ++#ifdef HAVE_ZSTD_H ++#include ++static sqfs_err sqfs_decompressor_zstd(void *in, size_t insz, ++ void *out, size_t *outsz) { ++ const size_t zstdout = ZSTD_decompress(out, *outsz, in, insz); ++ if (ZSTD_isError(zstdout)) ++ return SQFS_ERR; ++ *outsz = zstdout; ++ return SQFS_OK; ++} ++#define CAN_DECOMPRESS_ZSTD 1 ++#endif ++ + sqfs_decompressor sqfs_decompressor_get(sqfs_compression_type type) { + switch (type) { + #ifdef CAN_DECOMPRESS_ZLIB +@@ -111,13 +124,16 @@ sqfs_decompressor sqfs_decompressor_get(sqfs_compression_type type) { + #endif + #ifdef CAN_DECOMPRESS_LZ4 + case LZ4_COMPRESSION: return &sqfs_decompressor_lz4; ++#endif ++#ifdef CAN_DECOMPRESS_ZSTD ++ case ZSTD_COMPRESSION: return &sqfs_decompressor_zstd; + #endif + default: return NULL; + } + } + + static char *const sqfs_compression_names[SQFS_COMP_MAX] = { +- NULL, "zlib", "lzma", "lzo", "xz", "lz4", ++ NULL, "zlib", "lzma", "lzo", "xz", "lz4", "zstd", + }; + + char *sqfs_compression_name(sqfs_compression_type type) { +@@ -141,4 +157,7 @@ void sqfs_compression_supported(sqfs_compression_type *types) { + #ifdef CAN_DECOMPRESS_LZ4 + types[i++] = LZ4_COMPRESSION; + #endif ++#ifdef CAN_DECOMPRESS_ZSTD ++ types[i++] = ZSTD_COMPRESSION; ++#endif + } +diff --git a/squashfs_fs.h b/squashfs_fs.h +index 7269c1f..e0ab1f4 100644 +--- a/squashfs_fs.h ++++ b/squashfs_fs.h +@@ -126,6 +126,7 @@ + #define LZO_COMPRESSION 3 + #define XZ_COMPRESSION 4 + #define LZ4_COMPRESSION 5 ++#define ZSTD_COMPRESSION 6 + + struct squashfs_super_block { + __le32 s_magic; diff --git a/tests/libappimage/desktop_integration/CMakeLists.txt b/tests/libappimage/desktop_integration/CMakeLists.txt index a5296f78..52e2c327 100644 --- a/tests/libappimage/desktop_integration/CMakeLists.txt +++ b/tests/libappimage/desktop_integration/CMakeLists.txt @@ -31,6 +31,7 @@ target_link_libraries( PRIVATE libsquashfuse PRIVATE libarchive PRIVATE xz + PRIVATE zstd PRIVATE libzlib PRIVATE XdgUtils::DesktopEntry PRIVATE XdgUtils::BaseDir diff --git a/tests/libappimage/legacy/test_libappimage.cpp b/tests/libappimage/legacy/test_libappimage.cpp index 86875f37..e9aa311c 100644 --- a/tests/libappimage/legacy/test_libappimage.cpp +++ b/tests/libappimage/legacy/test_libappimage.cpp @@ -303,7 +303,7 @@ bool test_compare_bytes(const char* buf1, const char* buf2, int size) { TEST_F(LibAppImageTest, appimage_type2_digest_md5) { char digest[16]; - char expectedDigest[] = {-75, -71, 106, -93, 122, 114, 7, 127, -40, 10, -115, -82, -73, 115, -19, 1}; + char expectedDigest[] = {(char) -75, (char) -71, 106, (char) -93, 122, 114, 7, 127, (char) -40, 10, (char) -115, (char) -82, (char) -73, 115, (char) -19, 1}; EXPECT_TRUE(appimage_type2_digest_md5(appImage_type_2_file_path.c_str(), digest)); EXPECT_PRED3(test_compare_bytes, digest, expectedDigest, 16);