Refactor IO "read" core logic

.github/workflows/CI.yml

@@ -25,12 +25,14 @@ jobs:
       working-directory: ./tests/RedisConfigs
       run: docker compose -f docker-compose.yml up -d --wait
     - name: Install .NET SDK
-      uses: actions/setup-dotnet@v3
+      uses: actions/setup-dotnet@v4
       with:
         dotnet-version: |
           6.0.x
           8.0.x
           10.0.x
+    - name: .NET Build (eng prebuild)
+      run: dotnet build eng/StackExchange.Redis.Build/StackExchange.Redis.Build.csproj -c Release /p:CI=true
     - name: .NET Build
       run: dotnet build Build.csproj -c Release /p:CI=true
     - name: StackExchange.Redis.Tests
@@ -132,6 +134,8 @@ jobs:
         redis-cli -p 26381 INFO SERVER | grep redis_version || echo "Failed to get version for port 26381"
       continue-on-error: true
 
+    - name: .NET Build (eng prebuild)
+      run: dotnet build eng/StackExchange.Redis.Build/StackExchange.Redis.Build.csproj -c Release /p:CI=true
     - name: .NET Build
       run: dotnet build Build.csproj -c Release /p:CI=true
     - name: StackExchange.Redis.Tests

.github/workflows/codeql.yml

@@ -38,7 +38,9 @@ jobs:
     - name: Setup .NET
       uses: actions/setup-dotnet@v4
       with:
-        dotnet-version: '9.0.x'
+        dotnet-version: |
+          8.0.x
+          10.0.x
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
@@ -52,6 +54,10 @@ jobs:
         # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
         # queries: security-extended,security-and-quality
 
+    - if: matrix.language == 'csharp'
+      name: .NET Build (eng prebuild)
+      run: dotnet build eng/StackExchange.Redis.Build/StackExchange.Redis.Build.csproj -c Release /p:CI=true
+
     - if: matrix.language == 'csharp'
       name: .NET Build
       run: dotnet build Build.csproj -c Release /p:CI=true

Build.csproj

@@ -1,5 +1,6 @@
 <Project Sdk="Microsoft.Build.Traversal/3.0.2">
   <ItemGroup>
+    <ProjectReference Include="eng\**\*.csproj" />
     <ProjectReference Include="src\**\*.csproj" />
     <ProjectReference Include="tests\**\*.csproj" />
     <ProjectReference Include="toys\**\*.csproj" />

Directory.Build.props

@@ -10,7 +10,7 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <CodeAnalysisRuleset>$(MSBuildThisFileDirectory)Shared.ruleset</CodeAnalysisRuleset>
     <MSBuildWarningsAsMessages>NETSDK1069</MSBuildWarningsAsMessages>
-    <NoWarn>$(NoWarn);NU5105;NU1507;SER001;SER002;SER003</NoWarn>
+    <NoWarn>$(NoWarn);NU5105;NU1507;SER001;SER002;SER003;SER004;SER005</NoWarn>
     <PackageReleaseNotes>https://stackexchange.github.io/StackExchange.Redis/ReleaseNotes</PackageReleaseNotes>
     <PackageProjectUrl>https://stackexchange.github.io/StackExchange.Redis/</PackageProjectUrl>
     <PackageLicenseExpression>MIT</PackageLicenseExpression>

Directory.Packages.props

@@ -10,6 +10,9 @@
     <PackageVersion Include="System.IO.Compression" Version="4.3.0" />
     <!-- note that this bumps System.Buffers, so is pinned in down-level in SE csproj --> 
     <PackageVersion Include="System.IO.Hashing" Version="10.0.2" />
+    <!-- for RESPite -->
+    <PackageVersion Include="System.Buffers" Version="4.6.1" />
+    <PackageVersion Include="System.Memory" Version="4.6.1" />
 
     <!-- For analyzers, tied to the consumer's build SDK; at the moment, that means "us" -->
     <PackageVersion Include="Microsoft.CodeAnalysis.CSharp" Version="4.12.0" />

StackExchange.Redis.sln

@@ -21,6 +21,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		Shared.ruleset = Shared.ruleset
 		version.json = version.json
 		tests\RedisConfigs\.docker\Redis\Dockerfile = tests\RedisConfigs\.docker\Redis\Dockerfile
+		.github\workflows\codeql.yml = .github\workflows\codeql.yml
 	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "RedisConfigs", "RedisConfigs", "{96E891CD-2ED7-4293-A7AB-4C6F5D8D2B05}"
@@ -127,6 +128,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "eng", "eng", "{5FA0958E-6EB
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StackExchange.Redis.Build", "eng\StackExchange.Redis.Build\StackExchange.Redis.Build.csproj", "{190742E1-FA50-4E36-A8C4-88AE87654340}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RESPite", "src\RESPite\RESPite.csproj", "{AEA77181-DDD2-4E43-828B-908C7460A12D}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "RESPite.Tests", "tests\RESPite.Tests\RESPite.Tests.csproj", "{1D324077-A15E-4EE2-9AD6-A9045636CEAC}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -189,6 +194,14 @@ Global
 		{190742E1-FA50-4E36-A8C4-88AE87654340}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{190742E1-FA50-4E36-A8C4-88AE87654340}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{190742E1-FA50-4E36-A8C4-88AE87654340}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AEA77181-DDD2-4E43-828B-908C7460A12D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AEA77181-DDD2-4E43-828B-908C7460A12D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AEA77181-DDD2-4E43-828B-908C7460A12D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AEA77181-DDD2-4E43-828B-908C7460A12D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{1D324077-A15E-4EE2-9AD6-A9045636CEAC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{1D324077-A15E-4EE2-9AD6-A9045636CEAC}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{1D324077-A15E-4EE2-9AD6-A9045636CEAC}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{1D324077-A15E-4EE2-9AD6-A9045636CEAC}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -212,6 +225,8 @@ Global
 		{69A0ACF2-DF1F-4F49-B554-F732DCA938A3} = {73A5C363-CA1F-44C4-9A9B-EF791A76BA6A}
 		{59889284-FFEE-82E7-94CB-3B43E87DA6CF} = {73A5C363-CA1F-44C4-9A9B-EF791A76BA6A}
 		{190742E1-FA50-4E36-A8C4-88AE87654340} = {5FA0958E-6EBD-45F4-808E-3447A293F96F}
+		{AEA77181-DDD2-4E43-828B-908C7460A12D} = {00CA0876-DA9F-44E8-B0DC-A88716BF347A}
+		{1D324077-A15E-4EE2-9AD6-A9045636CEAC} = {73A5C363-CA1F-44C4-9A9B-EF791A76BA6A}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {193AA352-6748-47C1-A5FC-C9AA6B5F000B}

StackExchange.Redis.sln.DotSettings

@@ -26,4 +26,5 @@
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=xreadgroup/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=xrevrange/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=zcard/@EntryIndexedValue">True</s:Boolean>
-	<s:Boolean x:Key="/Default/UserDictionary/Words/=zscan/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=zscan/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=zset/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

docs/exp/SER004.md

@@ -0,0 +1,15 @@
+# RESPite
+
+RESPite is an experimental library that provides high-performance low-level RESP (Redis, etc) parsing and serialization.
+It is used as the IO core for StackExchange.Redis v3+. You should not (yet) use it directly unless you have a very
+good reason to do so.
+
+```xml
+<NoWarn>$(NoWarn);SER004</NoWarn>
+```
+
+or more granularly / locally in C#:
+
+``` c#
+#pragma warning disable SER004
+```

docs/exp/SER005.md

@@ -0,0 +1,21 @@
+# Unit Testing
+
+Unit testing is great! Yay, do more of that!
+
+This type is provided for external unit testing, in particular by people using modules or server features
+not directly implemented by SE.Redis - for example to verify messsage parsing or formatting without
+talking to a RESP server.
+
+These types are considered slightly more... *mercurial*. We encourage you to use them, but *occasionally*
+(not just for fun) you might need to update your test code if we tweak something. This should not impact
+"real" library usage.
+
+```xml
+<NoWarn>$(NoWarn);SER005</NoWarn>
+```
+
+or more granularly / locally in C#:
+
+``` c#
+#pragma warning disable SER005
+```

eng/StackExchange.Redis.Build/AsciiHash.md

@@ -0,0 +1,173 @@
+# AsciiHash
+
+Efficient matching of well-known short string tokens is a high-volume scenario, for example when matching RESP literals.
+
+The purpose of this generator is to efficiently interpret input tokens like `bin`, `f32`, etc - whether as byte or character data.
+
+There are multiple ways of using this tool, with the main distinction being whether you are confirming a single
+token, or choosing between multiple tokens (in which case an `enum` is more appropriate):
+
+## Isolated literals (part 1)
+
+When using individual tokens, a `static partial class` can be used to generate helpers:
+
+``` c#
+[AsciiHash] public static partial class bin { }
+[AsciiHash] public static partial class f32 { }
+```
+
+Usually the token is inferred from the name; `[AsciiHash("real value")]` can be used if the token is not a valid identifier.
+Underscores are replaced with hyphens, so a field called `my_token` has the default value `"my-token"`.
+The generator demands *all* of `[AsciiHash] public static partial class`, and note that any *containing* types must
+*also* be declared `partial`.
+
+The output is of the form:
+
+``` c#
+static partial class bin
+{
+    public const int Length = 3;
+    public const long HashCS = ...
+    public const long HashUC = ...
+    public static ReadOnlySpan<byte> U8 => @"bin"u8;
+    public static string Text => @"bin";
+    public static bool IsCS(in ReadOnlySpan<byte> value, long cs) => ...
+    public static bool IsCI(in RawResult value, long uc) => ...
+
+}
+```
+The `CS` and `UC` are case-sensitive and case-insensitive (using upper-case) tools, respectively.
+
+(this API is strictly an internal implementation detail, and can change at any time)
+
+This generated code allows for fast, efficient, and safe matching of well-known tokens, for example:
+
+``` c#
+var key = ...
+var hash = key.HashCS();
+switch (key.Length)
+{
+    case bin.Length when bin.Is(key, hash):
+        // handle bin
+        break;
+    case f32.Length when f32.Is(key, hash):
+        // handle f32
+        break;
+}
+```
+
+The switch on the `Length` is optional, but recommended - these low values can often be implemented (by the compiler)
+as a simple jump-table, which is very fast. However, switching on the hash itself is also valid. All hash matches
+must also perform a sequence equality check - the `Is(value, hash)` convenience method validates both hash and equality.
+
+Note that `switch` requires `const` values, hence why we use generated *types* rather than partial-properties
+that emit an instance with the known values. Also, the `"..."u8` syntax emits a span which is awkward to store, but
+easy to return via a property.
+
+## Isolated literals (part 2)
+
+In some cases, you want to be able to say "match this value, only known at runtime". For this, note that `AsciiHash`
+is also a `struct` that you can create an instance of and supply to code; the best way to do this is *inside* your
+`partial class`:
+
+``` c#
+[AsciiHash]
+static partial class bin
+{
+    public static readonly AsciiHash Hash = new(U8);
+}
+```
+
+Now, `bin.Hash` can be supplied to a caller that takes an `AsciiHash` instance (commonly with `in` semantics),
+which then has *instance* methods for case-sensitive and case-insensitive matching; the instance already knows
+the target hash and payload values.
+
+The `AsciiHash` returned implements `IEquatable<AsciiHash>` implementing case-sensitive equality; there are
+also independent case-sensitive and case-insensitive comparers available via the static
+`CaseSensitiveEqualityComparer` and `CaseInsensitiveEqualityComparer` properties respectively.
+
+Comparison values can be constructed on the fly on top of transient buffers using the constructors **that take
+arrays**. Note that the other constructors may allocate on a per-usage basis.
+
+## Enum parsing (part 1)
+
+When identifying multiple values, an `enum` may be more convenient. Consider:
+
+``` c#
+[AsciiHash]
+public static partial bool TryParse(ReadOnlySpan<byte> value, out SomeEnum value);
+```
+
+This generates an efficient parser; inputs can be common `byte` or `char` types. Case sensitivity
+is controlled by the optional `CaseSensitive` property on the attribute, or via a 3rd (`bool`) parameter
+bbon the method, i.e.
+
+``` c#
+[AsciiHash(CaseSensitive = false)]
+public static partial bool TryParse(ReadOnlySpan<byte> value, out SomeEnum value);
+```
+
+or
+
+``` c#
+[AsciiHash]
+public static partial bool TryParse(ReadOnlySpan<byte> value, out SomeEnum value, bool caseSensitive = true);
+```
+
+Individual enum members can also be marked with `[AsciiHash("token value")]` to override the token payload. If
+an enum member declares an empty explicit value (i.e. `[AsciiHash("")]`), then that member is ignored by the
+tool; this is useful for marking "unknown" or "invalid" enum values (commonly the first enum, which by
+convention has the value `0`):
+
+``` c#
+public enum SomeEnum
+{
+    [AsciiHash("")]
+    Unknown,
+    SomeRealValue,
+    [AsciiHash("another-real-value")]
+    AnotherRealValue,
+    // ...
+}
+```
+
+## Enum parsing (part 2)
+
+The tool has an *additional* facility when it comes to enums; you generally don't want to have to hard-code
+things like buffer-lengths into your code, but when parsing an enum, you need to know how many bytes to read.
+
+The tool can generate a `static partial class` that contains the maximum length of any token in the enum, as well
+as the maximum length of any token in bytes (when encoded as UTF-8). For example:
+
+``` c#
+[AsciiHash("SomeTypeName")]
+public enum SomeEnum
+{
+    // ...
+}
+```
+
+This generates a class like the following:
+
+``` c#
+static partial class SomeTypeName
+{
+    public const int EnumCount = 48;
+    public const int MaxChars = 11;
+    public const int MaxBytes = 11; // as UTF8
+    public const int BufferBytes = 16;
+}
+```
+
+The last of these is probably the most useful - it allows an additional byte (to rule out false-positives),
+and rounds up to word-sizes, allowing for convenient stack-allocation - for example:
+
+``` c#
+var span = reader.TryGetSpan(out var tmp) ? tmp : reader.Buffer(stackalloc byte[SomeTypeName.BufferBytes]);
+if (TryParse(span, out var value))
+{
+    // got a value
+}
+```
+
+which allows for very efficient parsing of well-known tokens.