Skip to content

Commit 3a08bc4

Browse files
ethomsonethomson
authored
Merge pull request libgit2#6235 from libgit2/ethomson/cli_benchmarks
Benchmarking suite
2 parents 8420ac4 + 4c6eab9 commit 3a08bc4

25 files changed

+1003
-7
lines changed

‎.github/workflows/benchmark.yml‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
# Benchmark libgit2 against the git reference implementation.
2+
name: Benchmark
3+
4+
on:
5+
workflow_dispatch:
6+
schedule:
7+
- cron: '15 4 * * *'
8+
9+
jobs:
10+
# Run our nightly builds. We build a matrix with the various build
11+
# targets and their details. Then we build either in a docker container
12+
# (Linux) or on the actual hosts (macOS, Windows).
13+
build:
14+
strategy:
15+
matrix:
16+
platform:
17+
- name: "Linux (clang, OpenSSL)"
18+
env:
19+
CC: clang
20+
CMAKE_OPTIONS: -DUSE_HTTPS=OpenSSL -DREGEX_BACKEND=builtin -DDEPRECATE_HARD=ON -DUSE_GSSAPI=ON -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI=ON -DCMAKE_BUILD_TYPE=Release
21+
CMAKE_BUILD_OPTIONS: --config Release
22+
id: linux
23+
os: ubuntu-latest
24+
setup-script: ubuntu
25+
- name: "macOS"
26+
os: macos-10.15
27+
env:
28+
CC: clang
29+
CMAKE_OPTIONS: -DREGEX_BACKEND=regcomp_l -DDEPRECATE_HARD=ON -DUSE_GSSAPI=ON -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI=ON -DCMAKE_BUILD_TYPE=Release
30+
CMAKE_BUILD_OPTIONS: --config Release
31+
PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig
32+
id: macos
33+
setup-script: osx
34+
- name: "Windows (amd64, Visual Studio)"
35+
os: windows-2019
36+
env:
37+
ARCH: amd64
38+
CMAKE_GENERATOR: Visual Studio 16 2019
39+
CMAKE_OPTIONS: -A x64 -DDEPRECATE_HARD=ON -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_CLI=ON -DCMAKE_BUILD_TYPE=Release
40+
CMAKE_BUILD_OPTIONS: --config Release
41+
id: windows
42+
setup-script: win32
43+
fail-fast: false
44+
name: "Build ${{ matrix.platform.name }}"
45+
env: ${{ matrix.platform.env }}
46+
runs-on: ${{ matrix.platform.os }}
47+
steps:
48+
- name: Check out repository
49+
uses: actions/checkout@v2
50+
with:
51+
path: source
52+
fetch-depth: 0
53+
- name: Set up benchmark environment
54+
run: source/ci/setup-${{ matrix.platform.setup-script }}-benchmark.sh
55+
shell: bash
56+
if: matrix.platform.setup-script != ''
57+
- name: Build
58+
run: |
59+
mkdir build && cd build
60+
../source/ci/build.sh
61+
shell: bash
62+
- name: Benchmark
63+
run: |
64+
if [[ "$(uname -s)" == MINGW* ]]; then
65+
GIT2_CLI="$(cygpath -w $(pwd))\\build\\Release\\git2_cli"
66+
else
67+
GIT2_CLI="$(pwd)/build/git2_cli"
68+
fi
69+
70+
mkdir benchmark && cd benchmark
71+
../source/tests/benchmarks/benchmark.sh --baseline-cli "git" --cli "${GIT2_CLI}" --json benchmarks.json --zip benchmarks.zip
72+
shell: bash
73+
- name: Upload results
74+
uses: actions/upload-artifact@v2
75+
with:
76+
name: benchmark-${{ matrix.platform.id }}
77+
path: benchmark
78+
if: always()

‎.github/workflows/main.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@ jobs:
214214
path: source
215215
fetch-depth: 0
216216
- name: Set up build environment
217-
run: source/ci/setup-${{ matrix.platform.setup-script }}.sh
217+
run: source/ci/setup-${{ matrix.platform.setup-script }}-build.sh
218218
shell: bash
219219
if: matrix.platform.setup-script != ''
220220
- name: Setup QEMU

‎.github/workflows/nightly.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -271,7 +271,7 @@ jobs:
271271
path: source
272272
fetch-depth: 0
273273
- name: Set up build environment
274-
run: source/ci/setup-${{ matrix.platform.setup-script }}.sh
274+
run: source/ci/setup-${{ matrix.platform.setup-script }}-build.sh
275275
shell: bash
276276
if: matrix.platform.setup-script != ''
277277
- name: Setup QEMU

‎ci/build.sh‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ echo "##########################################################################
5959
echo "## Configuring build environment"
6060
echo "##############################################################################"
6161

62-
echo cmake -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -G \"${CMAKE_GENERATOR}\" ${CMAKE_OPTIONS} -S \"${SOURCE_DIR}\"
62+
echo "${CMAKE}" -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -G \"${CMAKE_GENERATOR}\" ${CMAKE_OPTIONS} -S \"${SOURCE_DIR}\"
6363
env PATH="${BUILD_PATH}" "${CMAKE}" -DENABLE_WERROR=ON -DBUILD_EXAMPLES=ON -DBUILD_FUZZERS=ON -DUSE_STANDALONE_FUZZERS=ON -G "${CMAKE_GENERATOR}" ${CMAKE_OPTIONS} -S "${SOURCE_DIR}"
6464

6565
echo ""
@@ -69,10 +69,11 @@ echo "##########################################################################
6969

7070
# Determine parallelism; newer cmake supports `--build --parallel` but
7171
# we cannot yet rely on that.
72-
if [ "${CMAKE_GENERATOR}" = "Unix Makefiles" -a "${CORES}" != "" ]; then
72+
if [ "${CMAKE_GENERATOR}" = "Unix Makefiles" -a "${CORES}" != "" -a "${CMAKE_BUILD_OPTIONS}" = "" ]; then
7373
BUILDER=(make -j ${CORES})
7474
else
75-
BUILDER=("${CMAKE}" --build .)
75+
BUILDER=("${CMAKE}" --build . ${CMAKE_BUILD_OPTIONS})
7676
fi
7777

78+
echo "${BUILDER[@]}"
7879
env PATH="${BUILD_PATH}" "${BUILDER[@]}"

ci/setup-mingw.sh renamed to ci/setup-mingw-build.sh

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1-
#!/bin/sh -e
1+
#!/bin/sh
2+
3+
set -ex
24

35
echo "##############################################################################"
46
echo "## Downloading mingw"

‎ci/setup-osx-benchmark.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/sh
2+
3+
set -ex
4+
5+
brew update
6+
brew install hyperfine

ci/setup-osx.sh renamed to ci/setup-osx-build.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#!/bin/sh
22

3-
set -x
3+
set -ex
44

55
brew update
66
brew install pkgconfig zlib curl openssl libssh2 ninja

‎ci/setup-ubuntu-benchmark.sh‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/bin/sh
2+
3+
set -ex
4+
5+
sudo apt-get update
6+
sudo apt-get install -y --no-install-recommends \
7+
cargo \
8+
cmake \
9+
gcc \
10+
git \
11+
krb5-user \
12+
libkrb5-dev \
13+
libssl-dev \
14+
libz-dev \
15+
make \
16+
ninja-build \
17+
pkgconf
18+
19+
wget https://github.com/sharkdp/hyperfine/releases/download/v1.12.0/hyperfine_1.12.0_amd64.deb
20+
sudo dpkg -i hyperfine_1.12.0_amd64.deb

‎ci/setup-win32-benchmark.sh‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/sh
2+
3+
set -ex
4+
5+
choco install hyperfine zip
6+
7+
CHOCO_PATH=$(mktemp -d)
8+
curl -L https://github.com/ethomson/PurgeStandbyList/releases/download/v1.0/purgestandbylist.1.0.0.nupkg -o "${CHOCO_PATH}/purgestandbylist.1.0.0.nupkg"
9+
choco install purgestandbylist -s $(cygpath -w "${CHOCO_PATH}")

‎tests/README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
These are the unit and integration tests for the libgit2 projects.
44

5+
* `benchmarks`
6+
These are benchmark tests that excercise the CLI.
57
* `clar`
68
This is [clar](https://github.com/clar-test/clar) the common test framework.
79
* `headertest`

0 commit comments

Comments
 (0)