From 63b584a82528f1b7066d5d58022ed9a0c3a412f9 Mon Sep 17 00:00:00 2001
From: Nick Desaulniers <ndesaulniers@google.com>
Date: Fri, 1 Jul 2022 10:56:46 -0700
Subject: [PATCH] llvm-project: link against jemalloc

The following symbols from musl's malloc-ng implementation were
appearing in a perf profile of a kernel build within the top 20 hottest
symbols by cycle count.

malloc-ng:
Samples: 589K of event 'cycles:ppu', Event count (approx.): 11053523397362
  Overhead  Shared Object              Symbol
+    1.57%  clang                     [.] __libc_malloc_impl
     0.69%  clang                     [.] __libc_free
+    0.65%  clang                     [.] alloc_slot
+    0.58%  clang                     [.] get_meta

$ /usr/bin/time -v make CC=clang LD=ld.lld -j72 -s
Command being timed: "make CC=clang LD=ld.lld -j72 -s"
  User time (seconds): 2801.78
  System time (seconds): 395.48
  Percent of CPU this job got: 5212%
  Elapsed (wall clock) time (h:mm:ss or m:ss): 1:01.33

jemalloc:
Samples: 569K of event 'cycles:ppu', Event count (approx.): 10681965442287
  Overhead  Shared Object              Symbol
+    0.55%  clang                     [.] free
+    0.41%  clang                     [.] malloc

$ /usr/bin/time -v make CC=clang LD=ld.lld -j72 -s
Command being timed: "make CC=clang LD=ld.lld -j72 -s"
User time (seconds): 2626.97
System time (seconds): 248.37
Percent of CPU this job got: 4774%
Elapsed (wall clock) time (h:mm:ss or m:ss): 1:00.22

And these symbols are no longer in the top 20 hottest symbols by cycle
count.

Replace musl's malloc with jemalloc. Nothing major to write home about,
but gives a ~2.5% speedup by cycle count (N=1).

Fixes #36
---
 llvm-project/Dockerfile.epoch3 | 14 ++++++++++++++
 llvm-project/stage3.cmake      |  2 +-
 2 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/llvm-project/Dockerfile.epoch3 b/llvm-project/Dockerfile.epoch3
index 7016408..6752f25 100644
--- a/llvm-project/Dockerfile.epoch3
+++ b/llvm-project/Dockerfile.epoch3
@@ -5,6 +5,7 @@ RUN wget --no-verbose https://git.kernel.org/torvalds/t/linux-5.18-rc6.tar.gz
 RUN wget --no-verbose https://musl.libc.org/releases/musl-1.2.3.tar.gz
 RUN wget --no-verbose https://zlib.net/zlib-1.2.12.tar.gz
 RUN wget --no-verbose https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.1/llvm-project-14.0.1.src.tar.xz
+RUN wget --no-verbose https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2
 
 FROM ${BASE} AS stage2
 FROM docker.io/alpine:edge AS stage3
@@ -60,6 +61,19 @@ RUN make -C ${ZLIB_DIR} -j$(nproc)
 RUN make -C ${ZLIB_DIR} -j$(nproc) install
 RUN apk del make
 
+### Jemalloc
+COPY --from=source jemalloc-5.3.0.tar.bz2 .
+RUN tar xf jemalloc-5.3.0.tar.bz2
+ARG JEMALLOC_DIR=jemalloc-5.3.0/build
+RUN mkdir -p ${JEMALLOC_DIR}
+RUN cd ${JEMALLOC_DIR} && \
+  CC=clang AR=llvm-ar NM=llvm-nm CPPFLAGS=${SYSROOT} LDFLAGS=${SYSROOT} \
+  ../configure --disable-libdl --prefix=/usr
+RUN apk add make
+RUN make -C ${JEMALLOC_DIR} -j$(nproc) build_lib_static
+RUN make -C ${JEMALLOC_DIR} -j$(nproc) DESTDIR=/sysroot install_lib_static
+RUN apk del make
+
 ### LLVM
 COPY --from=source llvm-project-14.0.1.src.tar.xz .
 RUN tar xf llvm-project-14.0.1.src.tar.xz && \
diff --git a/llvm-project/stage3.cmake b/llvm-project/stage3.cmake
index 2a20874..4dedb42 100644
--- a/llvm-project/stage3.cmake
+++ b/llvm-project/stage3.cmake
@@ -13,7 +13,7 @@ set(CMAKE_CXX_FLAGS "--sysroot=/sysroot" CACHE STRING "")
 set(CMAKE_C_FLAGS "--sysroot=/sysroot" CACHE STRING "")
 
 # Statically link resulting executable.
-set(CMAKE_EXE_LINKER_FLAGS "-static -lc++abi" CACHE STRING "")
+set(CMAKE_EXE_LINKER_FLAGS "-static -lc++abi -ljemalloc" CACHE STRING "")
 
 # The compiler builtins are necessary.
 set(COMPILER_RT_BUILD_BUILTINS ON CACHE BOOL "")