feat: QoL: Byte Packing of integers [MTT-4924] (#2276)

Adds byte packing for most integer fields in the SDK (a few, such as hashes, are left un-packed because they're likely to be large integers and not benefit - and may possibly actually be harmed - by variable-length encoding). Integers in `RPC`s and `NetworkVariable`s are also packed properly. No functionality has been added to allow a value to be sent unpacked in `RPC`s or `NetworkVariable`s in cases where packing may be detrimental, but this can be achieved as follows:

```csharp
public struct UnpackedInt32 : INetworkSerializeByMemcpy
{
    int Value;
}

public NetworkVariable<UnpackedInt32> MyVariable;
````

In addition to byte packing most integer fields, this commit updates `WriteValueBitPacked()` to be able to handle the full range of its type (now using an extra byte - and thus ending up making the representation larger instead of smaller - for values at the extreme ends of the size), at the cost of using one extra bit to encode the size of the type.

Author: ShadauxCat

com.unity.netcode.gameobjects/CHANGELOG.md

@@ -13,6 +13,10 @@ Additional documentation and release notes are available at [Multiplayer Documen
 - Added `NetworkObject` auto-add helper and Multiplayer Tools install reminder settings to Project Settings. (#2285)
 - Added `public string DisconnectReason` getter to `NetworkManager` and `string Reason` to `ConnectionApprovalResponse`. Allows connection approval to communicate back a reason. Also added `public void DisconnectClient(ulong clientId, string reason)` allowing setting a disconnection reason, when explicitly disconnecting a client. 
 
+### Changed
+
+- Optimized bandwidth usage by encoding most integer fields using variable-length encoding. (#2276)
+
 ### Fixed
 
 - Fixed issue where the host would receive more than one event completed notification when loading or unloading a scene only when no clients were connected. (#2292)

com.unity.netcode.gameobjects/Editor/CodeGen/NetworkBehaviourILPP.cs

@@ -139,6 +139,19 @@ private bool IsMemcpyableType(TypeReference type)
             return false;
         }
 
+        private bool IsSpecialCaseType(TypeReference type)
+        {
+            foreach (var supportedType in SpecialCaseTypes)
+            {
+                if (type.FullName == supportedType.FullName)
+                {
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
         private void CreateNetworkVariableTypeInitializers(AssemblyDefinition assembly)
         {
             foreach (var typeDefinition in assembly.MainModule.Types)
@@ -153,6 +166,11 @@ private void CreateNetworkVariableTypeInitializers(AssemblyDefinition assembly)
 
                     foreach (var type in m_WrappedNetworkVariableTypes)
                     {
+                        if (IsSpecialCaseType(type))
+                        {
+                            continue;
+                        }
+
                         // If a serializable type isn't found, FallbackSerializer will be used automatically, which will
                         // call into UserNetworkVariableSerialization, giving the user a chance to define their own serializaiton
                         // for types that aren't in our official supported types list.
@@ -257,6 +275,20 @@ private void CreateNetworkVariableTypeInitializers(AssemblyDefinition assembly)
         private MethodReference m_NetworkVariableSerializationTypes_InitializeEqualityChecker_UnmanagedValueEquals_MethodRef;
         private MethodReference m_NetworkVariableSerializationTypes_InitializeEqualityChecker_ManagedClassEquals_MethodRef;
 
+        private MethodReference m_BytePacker_WriteValueBitPacked_Short_MethodRef;
+        private MethodReference m_BytePacker_WriteValueBitPacked_UShort_MethodRef;
+        private MethodReference m_BytePacker_WriteValueBitPacked_Int_MethodRef;
+        private MethodReference m_BytePacker_WriteValueBitPacked_UInt_MethodRef;
+        private MethodReference m_BytePacker_WriteValueBitPacked_Long_MethodRef;
+        private MethodReference m_BytePacker_WriteValueBitPacked_ULong_MethodRef;
+
+        private MethodReference m_ByteUnpacker_ReadValueBitPacked_Short_MethodRef;
+        private MethodReference m_ByteUnpacker_ReadValueBitPacked_UShort_MethodRef;
+        private MethodReference m_ByteUnpacker_ReadValueBitPacked_Int_MethodRef;
+        private MethodReference m_ByteUnpacker_ReadValueBitPacked_UInt_MethodRef;
+        private MethodReference m_ByteUnpacker_ReadValueBitPacked_Long_MethodRef;
+        private MethodReference m_ByteUnpacker_ReadValueBitPacked_ULong_MethodRef;
+
         private TypeReference m_FastBufferWriter_TypeRef;
         private readonly Dictionary<string, MethodReference> m_FastBufferWriter_WriteValue_MethodRefs = new Dictionary<string, MethodReference>();
         private readonly List<MethodReference> m_FastBufferWriter_ExtensionMethodRefs = new List<MethodReference>();
@@ -276,12 +308,13 @@ private void CreateNetworkVariableTypeInitializers(AssemblyDefinition assembly)
             typeof(decimal),
             typeof(double),
             typeof(float),
-            typeof(int),
+            // the following types have special handling
+            /*typeof(int),
             typeof(uint),
             typeof(long),
             typeof(ulong),
             typeof(short),
-            typeof(ushort),
+            typeof(ushort),*/
             typeof(Vector2),
             typeof(Vector3),
             typeof(Vector2Int),
@@ -293,6 +326,16 @@ private void CreateNetworkVariableTypeInitializers(AssemblyDefinition assembly)
             typeof(Ray),
             typeof(Ray2D)
         };
+        internal static readonly Type[] SpecialCaseTypes = new[]
+        {
+            // the following types have special handling
+            typeof(int),
+            typeof(uint),
+            typeof(long),
+            typeof(ulong),
+            typeof(short),
+            typeof(ushort),
+        };
 
         private const string k_Debug_LogError = nameof(Debug.LogError);
         private const string k_NetworkManager_LocalClientId = nameof(NetworkManager.LocalClientId);
@@ -343,6 +386,8 @@ private bool ImportReferences(ModuleDefinition moduleDefinition)
             TypeDefinition fastBufferWriterTypeDef = null;
             TypeDefinition fastBufferReaderTypeDef = null;
             TypeDefinition networkVariableSerializationTypesTypeDef = null;
+            TypeDefinition bytePackerTypeDef = null;
+            TypeDefinition byteUnpackerTypeDef = null;
             foreach (var netcodeTypeDef in m_NetcodeModule.GetAllTypes())
             {
                 if (networkManagerTypeDef == null && netcodeTypeDef.Name == nameof(NetworkManager))
@@ -398,6 +443,18 @@ private bool ImportReferences(ModuleDefinition moduleDefinition)
                     networkVariableSerializationTypesTypeDef = netcodeTypeDef;
                     continue;
                 }
+
+                if (bytePackerTypeDef == null && netcodeTypeDef.Name == nameof(BytePacker))
+                {
+                    bytePackerTypeDef = netcodeTypeDef;
+                    continue;
+                }
+
+                if (byteUnpackerTypeDef == null && netcodeTypeDef.Name == nameof(ByteUnpacker))
+                {
+                    byteUnpackerTypeDef = netcodeTypeDef;
+                    continue;
+                }
             }
 
             foreach (var methodDef in debugTypeDef.Methods)
@@ -652,6 +709,82 @@ private bool ImportReferences(ModuleDefinition moduleDefinition)
                 }
             }
 
+            foreach (var method in bytePackerTypeDef.Methods)
+            {
+                if (!method.IsStatic)
+                {
+                    continue;
+                }
+
+                switch (method.Name)
+                {
+                    case nameof(BytePacker.WriteValueBitPacked):
+                        if (method.Parameters[1].ParameterType.FullName == typeof(short).FullName)
+                        {
+                            m_BytePacker_WriteValueBitPacked_Short_MethodRef = m_MainModule.ImportReference(method);
+                        }
+                        else if (method.Parameters[1].ParameterType.FullName == typeof(ushort).FullName)
+                        {
+                            m_BytePacker_WriteValueBitPacked_UShort_MethodRef = m_MainModule.ImportReference(method);
+                        }
+                        else if (method.Parameters[1].ParameterType.FullName == typeof(int).FullName)
+                        {
+                            m_BytePacker_WriteValueBitPacked_Int_MethodRef = m_MainModule.ImportReference(method);
+                        }
+                        else if (method.Parameters[1].ParameterType.FullName == typeof(uint).FullName)
+                        {
+                            m_BytePacker_WriteValueBitPacked_UInt_MethodRef = m_MainModule.ImportReference(method);
+                        }
+                        else if (method.Parameters[1].ParameterType.FullName == typeof(long).FullName)
+                        {
+                            m_BytePacker_WriteValueBitPacked_Long_MethodRef = m_MainModule.ImportReference(method);
+                        }
+                        else if (method.Parameters[1].ParameterType.FullName == typeof(ulong).FullName)
+                        {
+                            m_BytePacker_WriteValueBitPacked_ULong_MethodRef = m_MainModule.ImportReference(method);
+                        }
+                        break;
+                }
+            }
+
+            foreach (var method in byteUnpackerTypeDef.Methods)
+            {
+                if (!method.IsStatic)
+                {
+                    continue;
+                }
+
+                switch (method.Name)
+                {
+                    case nameof(ByteUnpacker.ReadValueBitPacked):
+                        if (method.Parameters[1].ParameterType.FullName == typeof(short).MakeByRefType().FullName)
+                        {
+                            m_ByteUnpacker_ReadValueBitPacked_Short_MethodRef = m_MainModule.ImportReference(method);
+                        }
+                        else if (method.Parameters[1].ParameterType.FullName == typeof(ushort).MakeByRefType().FullName)
+                        {
+                            m_ByteUnpacker_ReadValueBitPacked_UShort_MethodRef = m_MainModule.ImportReference(method);
+                        }
+                        else if (method.Parameters[1].ParameterType.FullName == typeof(int).MakeByRefType().FullName)
+                        {
+                            m_ByteUnpacker_ReadValueBitPacked_Int_MethodRef = m_MainModule.ImportReference(method);
+                        }
+                        else if (method.Parameters[1].ParameterType.FullName == typeof(uint).MakeByRefType().FullName)
+                        {
+                            m_ByteUnpacker_ReadValueBitPacked_UInt_MethodRef = m_MainModule.ImportReference(method);
+                        }
+                        else if (method.Parameters[1].ParameterType.FullName == typeof(long).MakeByRefType().FullName)
+                        {
+                            m_ByteUnpacker_ReadValueBitPacked_Long_MethodRef = m_MainModule.ImportReference(method);
+                        }
+                        else if (method.Parameters[1].ParameterType.FullName == typeof(ulong).MakeByRefType().FullName)
+                        {
+                            m_ByteUnpacker_ReadValueBitPacked_ULong_MethodRef = m_MainModule.ImportReference(method);
+                        }
+                        break;
+                }
+            }
+
             return true;
         }
 
@@ -1008,6 +1141,36 @@ private MethodReference GetFastBufferWriterWriteMethod(string name, TypeReferenc
 
         private bool GetWriteMethodForParameter(TypeReference paramType, out MethodReference methodRef)
         {
+            if (paramType.FullName == typeof(short).FullName)
+            {
+                methodRef = m_BytePacker_WriteValueBitPacked_Short_MethodRef;
+                return true;
+            }
+            if (paramType.FullName == typeof(ushort).FullName)
+            {
+                methodRef = m_BytePacker_WriteValueBitPacked_UShort_MethodRef;
+                return true;
+            }
+            if (paramType.FullName == typeof(int).FullName)
+            {
+                methodRef = m_BytePacker_WriteValueBitPacked_Int_MethodRef;
+                return true;
+            }
+            if (paramType.FullName == typeof(uint).FullName)
+            {
+                methodRef = m_BytePacker_WriteValueBitPacked_UInt_MethodRef;
+                return true;
+            }
+            if (paramType.FullName == typeof(long).FullName)
+            {
+                methodRef = m_BytePacker_WriteValueBitPacked_Long_MethodRef;
+                return true;
+            }
+            if (paramType.FullName == typeof(ulong).FullName)
+            {
+                methodRef = m_BytePacker_WriteValueBitPacked_ULong_MethodRef;
+                return true;
+            }
             var assemblyQualifiedName = paramType.FullName + ", " + paramType.Resolve().Module.Assembly.FullName;
             var foundMethodRef = m_FastBufferWriter_WriteValue_MethodRefs.TryGetValue(assemblyQualifiedName, out methodRef);
 
@@ -1154,6 +1317,36 @@ private MethodReference GetFastBufferReaderReadMethod(string name, TypeReference
 
         private bool GetReadMethodForParameter(TypeReference paramType, out MethodReference methodRef)
         {
+            if (paramType.FullName == typeof(short).FullName)
+            {
+                methodRef = m_ByteUnpacker_ReadValueBitPacked_Short_MethodRef;
+                return true;
+            }
+            if (paramType.FullName == typeof(ushort).FullName)
+            {
+                methodRef = m_ByteUnpacker_ReadValueBitPacked_UShort_MethodRef;
+                return true;
+            }
+            if (paramType.FullName == typeof(int).FullName)
+            {
+                methodRef = m_ByteUnpacker_ReadValueBitPacked_Int_MethodRef;
+                return true;
+            }
+            if (paramType.FullName == typeof(uint).FullName)
+            {
+                methodRef = m_ByteUnpacker_ReadValueBitPacked_UInt_MethodRef;
+                return true;
+            }
+            if (paramType.FullName == typeof(long).FullName)
+            {
+                methodRef = m_ByteUnpacker_ReadValueBitPacked_Long_MethodRef;
+                return true;
+            }
+            if (paramType.FullName == typeof(ulong).FullName)
+            {
+                methodRef = m_ByteUnpacker_ReadValueBitPacked_ULong_MethodRef;
+                return true;
+            }
             var assemblyQualifiedName = paramType.FullName + ", " + paramType.Resolve().Module.Assembly.FullName;
 
             var foundMethodRef = m_FastBufferReader_ReadValue_MethodRefs.TryGetValue(assemblyQualifiedName, out methodRef);

com.unity.netcode.gameobjects/Runtime/Core/NetworkBehaviour.cs

@@ -21,6 +21,7 @@ internal enum __RpcExecStage
             Client = 2
         }
 
+
         // NetworkBehaviourILPP will override this in derived classes to return the name of the concrete type
         internal virtual string __getTypeName() => nameof(NetworkBehaviour);
 
@@ -776,6 +777,11 @@ internal void WriteNetworkVariableData(FastBufferWriter writer, ulong targetClie
                 if (canClientRead)
                 {
                     var writePos = writer.Position;
+                    // Note: This value can't be packed because we don't know how large it will be in advance
+                    // we reserve space for it, then write the data, then come back and fill in the space
+                    // to pack here, we'd have to write data to a temporary buffer and copy it in - which
+                    // isn't worth possibly saving one byte if and only if the data is less than 63 bytes long...
+                    // The way we do packing, any value > 63 in a ushort will use the full 2 bytes to represent.
                     writer.WriteValueSafe((ushort)0);
                     var startPos = writer.Position;
                     NetworkVariableFields[j].WriteField(writer);

com.unity.netcode.gameobjects/Runtime/Core/NetworkManager.cs

@@ -2185,14 +2185,11 @@ internal void HandleConnectionApproval(ulong ownerClientId, ConnectionApprovalRe
                     // Generate a SceneObject for the player object to spawn
                     var sceneObject = new NetworkObject.SceneObject
                     {
-                        Header = new NetworkObject.SceneObject.HeaderData
-                        {
-                            IsPlayerObject = true,
-                            OwnerClientId = ownerClientId,
-                            IsSceneObject = false,
-                            HasTransform = true,
-                            Hash = playerPrefabHash,
-                        },
+                        OwnerClientId = ownerClientId,
+                        IsPlayerObject = true,
+                        IsSceneObject = false,
+                        HasTransform = true,
+                        Hash = playerPrefabHash,
                         TargetClientId = ownerClientId,
                         Transform = new NetworkObject.SceneObject.TransformData
                         {
@@ -2309,11 +2306,11 @@ internal void ApprovedPlayerSpawn(ulong clientId, uint playerPrefabHash)
                 {
                     ObjectInfo = ConnectedClients[clientId].PlayerObject.GetMessageSceneObject(clientPair.Key)
                 };
-                message.ObjectInfo.Header.Hash = playerPrefabHash;
-                message.ObjectInfo.Header.IsSceneObject = false;
-                message.ObjectInfo.Header.HasParent = false;
-                message.ObjectInfo.Header.IsPlayerObject = true;
-                message.ObjectInfo.Header.OwnerClientId = clientId;
+                message.ObjectInfo.Hash = playerPrefabHash;
+                message.ObjectInfo.IsSceneObject = false;
+                message.ObjectInfo.HasParent = false;
+                message.ObjectInfo.IsPlayerObject = true;
+                message.ObjectInfo.OwnerClientId = clientId;
                 var size = SendMessage(ref message, NetworkDelivery.ReliableFragmentedSequenced, clientPair.Key);
                 NetworkMetrics.TrackObjectSpawnSent(clientPair.Key, ConnectedClients[clientId].PlayerObject, size);
             }