Skip to content

Commit 6a917c0

Browse files
lhchavezlhchavez
committed
Add CI support for Memory and UndefinedBehavior Sanitizers
This change adds two new build targets: MSan and UBSan. This is because even though OSS-Fuzz is great and adds a lot of coverage, it only does that for the fuzz targets, so the rest of the codebase is not necessarily run with the Sanitizers ever :( So this change makes sure that MSan/UBSan warnings don't make it into the codebase. As part of this change, the Ubuntu focal container is introduced. It builds mbedTLS and libssh2 as debug libraries into /usr/local and as MSan-enabled libraries into /usr/local/msan. This latter part is needed because MSan requires the binary and all its dependent libraries to be built with MSan support so that memory allocations and deallocations are tracked correctly to avoid false positives.
1 parent 325375e commit 6a917c0

File tree

4 files changed

+138
-8
lines changed

4 files changed

+138
-8
lines changed

‎.github/workflows/main.yml‎

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ jobs:
2828
container:
2929
- xenial
3030
- bionic
31+
- focal
3132
- docurium
3233
runs-on: ubuntu-latest
3334
steps:
@@ -86,6 +87,26 @@ jobs:
8687
CMAKE_OPTIONS: -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_LEAK_CHECKER=valgrind -DUSE_GSSAPI=ON
8788
CMAKE_GENERATOR: Ninja
8889
os: ubuntu-latest
90+
- # Focal, Clang 10, mbedTLS, MemorySanitizer
91+
image: focal
92+
env:
93+
CC: clang-10
94+
CFLAGS: -fsanitize=memory -fsanitize-memory-track-origins=2 -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer
95+
CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local/msan -DUSE_HTTPS=mbedTLS -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON
96+
CMAKE_GENERATOR: Ninja
97+
SKIP_SSH_TESTS: true
98+
ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10
99+
os: ubuntu-latest
100+
- # Focal, Clang 10, OpenSSL, UndefinedBehaviorSanitizer
101+
image: focal
102+
env:
103+
CC: clang-10
104+
CFLAGS: -fsanitize=undefined,nullability -fno-sanitize-recover=undefined,nullability -fsanitize-blacklist=/home/libgit2/source/script/sanitizers.supp -fno-optimize-sibling-calls -fno-omit-frame-pointer
105+
CMAKE_OPTIONS: -DCMAKE_PREFIX_PATH=/usr/local -DUSE_HTTPS=OpenSSL -DUSE_SHA1=HTTPS -DREGEX_BACKEND=pcre -DDEPRECATE_HARD=ON -DUSE_BUNDLED_ZLIB=ON
106+
CMAKE_GENERATOR: Ninja
107+
SKIP_SSH_TESTS: true
108+
ASAN_SYMBOLIZER_PATH: /usr/bin/llvm-symbolizer-10
109+
os: ubuntu-latest
89110
- # macOS
90111
os: macos-10.15
91112
env:
@@ -161,7 +182,21 @@ jobs:
161182
export GITTEST_NEGOTIATE_PASSWORD="${{ secrets.GITTEST_NEGOTIATE_PASSWORD }}"
162183
163184
if [ -n "${{ matrix.platform.image }}" ]; then
164-
docker run -v $(pwd):/home/libgit2/source -w /home/libgit2/source -e CC -e CMAKE_GENERATOR -e CMAKE_OPTIONS -e PKG_CONFIG_PATH -e GITTEST_NEGOTIATE_PASSWORD -e SKIP_SSH_TESTS -e SKIP_NEGOTIATE_TESTS ${{ env.docker-registry-container-sha }} /bin/bash -c "mkdir build && cd build && ../azure-pipelines/build.sh && ../azure-pipelines/test.sh"
185+
docker run \
186+
--rm \
187+
-v "$(pwd):/home/libgit2/source" \
188+
-w /home/libgit2/source \
189+
-e ASAN_SYMBOLIZER_PATH \
190+
-e CC \
191+
-e CFLAGS \
192+
-e CMAKE_GENERATOR \
193+
-e CMAKE_OPTIONS \
194+
-e GITTEST_NEGOTIATE_PASSWORD \
195+
-e PKG_CONFIG_PATH \
196+
-e SKIP_NEGOTIATE_TESTS \
197+
-e SKIP_SSH_TESTS \
198+
${{ env.docker-registry-container-sha }} \
199+
/bin/bash -c "mkdir build && cd build && ../azure-pipelines/build.sh && ../azure-pipelines/test.sh"
165200
else
166201
mkdir build && cd build
167202
../azure-pipelines/build.sh
@@ -189,7 +224,12 @@ jobs:
189224
git config user.email 'libgit2@users.noreply.github.com'
190225
git branch gh-pages origin/gh-pages
191226
docker login https://${{ env.docker-registry }} -u ${{ github.actor }} -p ${{ github.token }}
192-
docker run --rm -v $(pwd):/home/libgit2/source -w /home/libgit2/source ${{ env.docker-registry }}/${{ github.repository }}/docurium:latest cm doc api.docurium
227+
docker run \
228+
--rm \
229+
-v "$(pwd):/home/libgit2/source" \
230+
-w /home/libgit2/source \
231+
${{ env.docker-registry }}/${{ github.repository }}/docurium:latest \
232+
cm doc api.docurium
193233
git checkout gh-pages
194234
zip --exclude .git/\* --exclude .gitignore --exclude .gitattributes -r api-documentation.zip .
195235
- uses: actions/upload-artifact@v2

‎azure-pipelines/build.sh‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -37,24 +37,31 @@ echo "Kernel version:"
3737
uname -a 2>&1 | indent
3838

3939
echo "CMake version:"
40-
env PATH="$BUILD_PATH" "$CMAKE" --version 2>&1 | indent
40+
env PATH="${BUILD_PATH}" "${CMAKE}" --version 2>&1 | indent
4141

42-
if test -n "$CC"; then
42+
if test -n "${CC}"; then
4343
echo "Compiler version:"
44-
"$CC" --version 2>&1 | indent
44+
"${CC}" --version 2>&1 | indent
45+
fi
46+
echo "Environment:"
47+
if test -n "${CC}"; then
48+
echo "CC=${CC}" | indent
49+
fi
50+
if test -n "${CFLAGS}"; then
51+
echo "CFLAGS=${CFLAGS}" | indent
4552
fi
4653
echo ""
4754

4855
echo "##############################################################################"
4956
echo "## Configuring build environment"
5057
echo "##############################################################################"
5158

52-
echo cmake ${SOURCE_DIR} -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -G \"${CMAKE_GENERATOR}\" ${CMAKE_OPTIONS}
53-
env PATH="$BUILD_PATH" "$CMAKE" ${SOURCE_DIR} -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -G "${CMAKE_GENERATOR}" ${CMAKE_OPTIONS}
59+
echo cmake -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -G \"${CMAKE_GENERATOR}\" ${CMAKE_OPTIONS} -S \"${SOURCE_DIR}\"
60+
env PATH="${BUILD_PATH}" "${CMAKE}" -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -G "${CMAKE_GENERATOR}" ${CMAKE_OPTIONS} -S "${SOURCE_DIR}"
5461

5562
echo ""
5663
echo "##############################################################################"
5764
echo "## Building libgit2"
5865
echo "##############################################################################"
5966

60-
env PATH="$BUILD_PATH" "$CMAKE" --build .
67+
env PATH="${BUILD_PATH}" "${CMAKE}" --build .

‎azure-pipelines/docker/focal‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
FROM ubuntu:focal AS apt
2+
RUN apt-get update && \
3+
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
4+
bzip2 \
5+
clang-10 \
6+
cmake \
7+
curl \
8+
gcc-10 \
9+
git \
10+
krb5-user \
11+
libcurl4-gnutls-dev \
12+
libgcrypt20-dev \
13+
libkrb5-dev \
14+
libpcre3-dev \
15+
libssl-dev \
16+
libz-dev \
17+
llvm-10 \
18+
make \
19+
ninja-build \
20+
openjdk-8-jre-headless \
21+
openssh-server \
22+
openssl \
23+
pkgconf \
24+
python \
25+
sudo \
26+
valgrind \
27+
&& \
28+
rm -rf /var/lib/apt/lists/* && \
29+
mkdir /usr/local/msan
30+
31+
FROM apt AS mbedtls
32+
RUN cd /tmp && \
33+
curl --location --silent --show-error https://tls.mbed.org/download/mbedtls-2.16.2-apache.tgz | \
34+
tar -xz && \
35+
cd mbedtls-2.16.2 && \
36+
scripts/config.pl unset MBEDTLS_AESNI_C && \
37+
scripts/config.pl set MBEDTLS_MD4_C 1 && \
38+
mkdir build build-msan && \
39+
cd build && \
40+
CC=clang-10 CFLAGS="-fPIC" cmake -G Ninja -DENABLE_PROGRAMS=OFF -DENABLE_TESTING=OFF -DUSE_SHARED_MBEDTLS_LIBRARY=ON -DUSE_STATIC_MBEDTLS_LIBRARY=OFF -DCMAKE_BUILD_TYPE=Debug -DCMAKE_PREFIX_PATH=/usr/local -DCMAKE_INSTALL_PREFIX=/usr/local .. && \
41+
ninja install && \
42+
cd ../build-msan && \
43+
CC=clang-10 CFLAGS="-fPIC" cmake -G Ninja -DENABLE_PROGRAMS=OFF -DENABLE_TESTING=OFF -DUSE_SHARED_MBEDTLS_LIBRARY=ON -DUSE_STATIC_MBEDTLS_LIBRARY=OFF -DCMAKE_BUILD_TYPE=MemSanDbg -DCMAKE_INSTALL_PREFIX=/usr/local/msan .. && \
44+
ninja install && \
45+
cd .. && \
46+
rm -rf mbedtls-2.16.2
47+
48+
FROM mbedtls AS libssh2
49+
RUN cd /tmp && \
50+
curl --insecure --location --silent --show-error https://www.libssh2.org/download/libssh2-1.8.2.tar.gz | \
51+
tar -xz && \
52+
cd libssh2-1.8.2 && \
53+
mkdir build build-msan && \
54+
cd build && \
55+
CC=clang-10 CFLAGS="-fPIC" cmake -G Ninja -DBUILD_SHARED_LIBS=ON -DCRYPTO_BACKEND=Libgcrypt -DCMAKE_PREFIX_PATH=/usr/local -DCMAKE_INSTALL_PREFIX=/usr/local .. && \
56+
ninja install && \
57+
cd ../build-msan && \
58+
CC=clang-10 CFLAGS="-fPIC -fsanitize=memory -fno-optimize-sibling-calls -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer" LDFLAGS="-fsanitize=memory" cmake -G Ninja -DBUILD_SHARED_LIBS=ON -DCRYPTO_BACKEND=mbedTLS -DCMAKE_PREFIX_PATH=/usr/local/msan -DCMAKE_INSTALL_PREFIX=/usr/local/msan .. && \
59+
ninja install && \
60+
cd .. && \
61+
rm -rf libssh2-1.8.2
62+
63+
FROM libssh2 AS valgrind
64+
RUN cd /tmp && \
65+
curl --insecure --location --silent --show-error https://sourceware.org/pub/valgrind/valgrind-3.15.0.tar.bz2 | \
66+
tar -xj && \
67+
cd valgrind-3.15.0 && \
68+
CC=clang-10 ./configure && \
69+
make MAKEFLAGS="-j -l$(grep -c ^processor /proc/cpuinfo)" && \
70+
make install && \
71+
cd .. && \
72+
rm -rf valgrind-3.15.0
73+
74+
FROM valgrind AS configure
75+
COPY entrypoint.sh /usr/local/bin/entrypoint.sh
76+
RUN chmod a+x /usr/local/bin/entrypoint.sh
77+
RUN mkdir /var/run/sshd
78+
79+
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]

‎script/sanitizers.supp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
[undefined]
2+
# This library allows unaligned access on Intel-like processors. Prevent UBSan
3+
# from complaining about that.
4+
fun:sha1_compression_states

0 commit comments

Comments
 (0)