Skip to content

Commit 2578916

Browse files
kzukzu
committed
Add benchmarks against jq executable
1 parent b8e15bf commit 2578916

7 files changed

Lines changed: 438 additions & 0 deletions

File tree

‎.github/design.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,12 @@ The public façade `Jq` exposes `Parse()` to obtain a cacheable `JqExpression`,
5252
jqsharp/
5353
├── JQSharp.slnx # Solution (src + tests)
5454
├── src/
55+
│ ├── Benchmarks/
56+
│ │ ├── Benchmarks.csproj # BenchmarkDotNet harness for jq.exe vs jqsharp comparisons
57+
│ │ ├── Program.cs # 1M-message WhatsApp benchmark entry point
58+
│ │ └── WhatsApp/
59+
│ │ ├── Message.jq # Benchmark query copied from Devlooped.WhatsApp
60+
│ │ └── Text.json # Benchmark sample payload copied from Devlooped.WhatsApp tests
5561
│ ├── JQSharp/
5662
│ │ ├── JQSharp.csproj # Library — net10.0, System.Text.Json only
5763
│ │ ├── Jq.cs # Public façade: Parse() and Evaluate()
@@ -91,6 +97,18 @@ jqsharp/
9197

9298
---
9399

100+
## 2.1 Benchmark Project
101+
102+
The repository also includes a small BenchmarkDotNet console project under `src/Benchmarks/` for throughput comparisons against the `Devlooped.JQ` NuGet package. The current benchmark uses the WhatsApp `Message.jq` filter and sample webhook payload from `Devlooped.WhatsApp`, then processes a 1,000,000-message workload in three modes:
103+
104+
- `Devlooped.JQ`: batched execution through the bundled `jq.exe` process wrapper
105+
- `Devlooped.JQSharp` without expression caching: `Jq.Evaluate(query, input)`
106+
- `Devlooped.JQSharp` with expression caching: `Jq.Parse(query)` once, then `JqExpression.Evaluate(input)`
107+
108+
To keep the benchmark self-contained and repeatable, the WhatsApp query and sample payload are copied into the benchmark project and shipped as content files.
109+
110+
---
111+
94112
## 3. Parser Design
95113

96114
`JqParser` is a **hand-written recursive-descent parser** that operates directly on the source string via a cursor (`position`). There is no separate lexer/tokenizer stage — the parser consumes characters inline, using helper methods like `Peek()`, `Consume()`, `TryConsume()`, `TryConsumeKeyword()`, etc.

‎JQSharp.slnx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
<Solution>
2+
<Project Path="src/Benchmarks/Benchmarks.csproj" />
23
<Project Path="src/JQSharp/JQSharp.csproj" />
34
<Project Path="src/Tests/Tests.csproj" />
45
</Solution>

‎readme.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,22 @@ foreach (var json in GetJsonStream())
6767
}
6868
```
6969

70+
### Performance vs jq.exe
71+
72+
```
73+
BenchmarkDotNet v0.15.8, Windows 11 (10.0.26200.8037/25H2/2025Update/HudsonValley2)
74+
AMD Ryzen AI 9 HX 370 w/ Radeon 890M 2.00GHz, 1 CPU, 24 logical and 12 physical cores
75+
.NET SDK 10.0.104
76+
[Host] : .NET 8.0.25 (8.0.25, 8.0.2526.11203), X64 RyuJIT x86-64-v4
77+
DefaultJob : .NET 8.0.25 (8.0.25, 8.0.2526.11203), X64 RyuJIT x86-64-v4
78+
```
79+
80+
| Method | Mean | Error | StdDev | Ratio | RatioSD | Gen0 | Gen1 | Allocated | Alloc Ratio |
81+
|-------------------------|-------------:|-----------:|-----------:|------:|--------:|--------:|-------:|----------:|------------:|
82+
| jq executable | 20,774.05 us | 415.148 us | 838.619 us | 1.002 | 0.06 | - | - | 122.23 KB | 1.00 |
83+
| JQSharp no caching | 149.67 us | 2.370 us | 2.911 us | 0.007 | 0.00 | 41.0156 | 8.3008 | 336.23 KB | 2.75 |
84+
| JQSharp w/query caching | 15.52 us | 0.061 us | 0.057 us | 0.001 | 0.00 | 6.7444 | 0.3357 | 55.34 KB | 0.45 |
85+
7086
### Transforming JSON
7187

7288
jq's full filter language is available, including pipes, array/object construction,

‎src/Benchmarks/Benchmarks.csproj‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
3+
<PropertyGroup>
4+
<OutputType>Exe</OutputType>
5+
<TargetFramework>net8.0</TargetFramework>
6+
<ImplicitUsings>enable</ImplicitUsings>
7+
<Nullable>enable</Nullable>
8+
<IsPackable>false</IsPackable>
9+
</PropertyGroup>
10+
11+
<ItemGroup>
12+
<PackageReference Include="BenchmarkDotNet" Version="0.15.8" />
13+
<PackageReference Include="Devlooped.JQ" Version="1.8.1.1" />
14+
</ItemGroup>
15+
16+
<ItemGroup>
17+
<ProjectReference Include="..\JQSharp\JQSharp.csproj" />
18+
</ItemGroup>
19+
20+
<ItemGroup>
21+
<None Update="WhatsApp\**\*">
22+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
23+
</None>
24+
</ItemGroup>
25+
26+
</Project>

‎src/Benchmarks/Program.cs‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
using System.Diagnostics;
2+
using System.Text.Json;
3+
using BenchmarkDotNet.Attributes;
4+
using BenchmarkDotNet.Running;
5+
using Devlooped;
6+
7+
if (Debugger.IsAttached)
8+
{
9+
await RunBenchmarksUnderDebuggerAsync();
10+
return;
11+
}
12+
13+
BenchmarkRunner.Run<WhatsAppMessageBenchmarks>();
14+
15+
static async Task RunBenchmarksUnderDebuggerAsync()
16+
{
17+
var benchmarks = new WhatsAppMessageBenchmarks();
18+
19+
benchmarks.Setup();
20+
21+
Console.WriteLine($"{nameof(WhatsAppMessageBenchmarks.DevloopedJQ)}: {await benchmarks.DevloopedJQ()}");
22+
Console.WriteLine($"{nameof(WhatsAppMessageBenchmarks.JQSharpWithoutExpressionCaching)}: {benchmarks.JQSharpWithoutExpressionCaching()}");
23+
Console.WriteLine($"{nameof(WhatsAppMessageBenchmarks.JQSharpWithExpressionCaching)}: {benchmarks.JQSharpWithExpressionCaching()}");
24+
}
25+
26+
[MemoryDiagnoser]
27+
public class WhatsAppMessageBenchmarks
28+
{
29+
byte[] messageJsonUtf8 = [];
30+
string messageJson = string.Empty;
31+
string query = string.Empty;
32+
JqExpression cachedExpression = default!;
33+
int expectedResultsPerMessage;
34+
35+
[GlobalSetup]
36+
public void Setup()
37+
{
38+
var basePath = AppContext.BaseDirectory;
39+
messageJsonUtf8 = File.ReadAllBytes(Path.Combine(basePath, "WhatsApp", "Text.json"));
40+
messageJson = File.ReadAllText(Path.Combine(basePath, "WhatsApp", "Text.json"));
41+
query = File.ReadAllText(Path.Combine(basePath, "WhatsApp", "Message.jq"));
42+
cachedExpression = Jq.Parse(query);
43+
44+
using (var document = JsonDocument.Parse(messageJsonUtf8))
45+
expectedResultsPerMessage = ConsumeResults(cachedExpression.Evaluate(document.RootElement));
46+
47+
if (expectedResultsPerMessage <= 0)
48+
throw new InvalidOperationException("The benchmark query did not produce any results for the sample WhatsApp payload.");
49+
50+
var wrapperSanity = JQ.ExecuteAsync(new JqParams(query)
51+
{
52+
Json = messageJson,
53+
RawOutput = false,
54+
CompactOutput = true,
55+
MonochromeOutput = true,
56+
}).GetAwaiter().GetResult();
57+
58+
if (wrapperSanity.ExitCode != 0)
59+
throw new InvalidOperationException($"Devlooped.JQ failed during setup: {wrapperSanity.StandardError}");
60+
61+
var wrapperResults = CountLines(wrapperSanity.StandardOutput);
62+
if (wrapperResults != expectedResultsPerMessage)
63+
throw new InvalidOperationException($"Expected {expectedResultsPerMessage} wrapper results but got {wrapperResults}.");
64+
}
65+
66+
[Benchmark(Baseline = true)]
67+
public async Task<int> DevloopedJQ()
68+
{
69+
var result = await JQ.ExecuteAsync(new JqParams(query)
70+
{
71+
Json = messageJson,
72+
RawOutput = false,
73+
CompactOutput = true,
74+
MonochromeOutput = true,
75+
});
76+
77+
if (result.ExitCode != 0)
78+
throw new InvalidOperationException(result.StandardError);
79+
80+
var outputCount = CountLines(result.StandardOutput);
81+
if (outputCount != expectedResultsPerMessage)
82+
throw new InvalidOperationException($"Expected {expectedResultsPerMessage} jq.exe results but got {outputCount}.");
83+
84+
return outputCount;
85+
}
86+
87+
[Benchmark]
88+
public int JQSharpWithoutExpressionCaching()
89+
{
90+
using var document = JsonDocument.Parse(messageJsonUtf8);
91+
var totalResults = ConsumeResults(Jq.Evaluate(query, document.RootElement));
92+
93+
if (totalResults != expectedResultsPerMessage)
94+
throw new InvalidOperationException($"Expected {expectedResultsPerMessage} uncached jqsharp results but got {totalResults}.");
95+
96+
return totalResults;
97+
}
98+
99+
[Benchmark]
100+
public int JQSharpWithExpressionCaching()
101+
{
102+
using var document = JsonDocument.Parse(messageJsonUtf8);
103+
var totalResults = ConsumeResults(cachedExpression.Evaluate(document.RootElement));
104+
105+
if (totalResults != expectedResultsPerMessage)
106+
throw new InvalidOperationException($"Expected {expectedResultsPerMessage} cached jqsharp results but got {totalResults}.");
107+
108+
return totalResults;
109+
}
110+
111+
static int ConsumeResults(IEnumerable<JsonElement> results)
112+
{
113+
var count = 0;
114+
115+
foreach (var result in results)
116+
{
117+
_ = result.ValueKind;
118+
count++;
119+
}
120+
121+
return count;
122+
}
123+
124+
static int CountLines(string output)
125+
{
126+
if (string.IsNullOrWhiteSpace(output))
127+
return 0;
128+
129+
var count = 1;
130+
131+
foreach (var ch in output)
132+
{
133+
if (ch == '\n')
134+
count++;
135+
}
136+
137+
return count;
138+
}
139+
}

0 commit comments

Comments
 (0)