Improve MemoryPackSerializer.Deserialize(Stream) performance

Cysharp · Nov 19, 2022 · 957c310 · 957c310
1 parent e462ea9
commit 957c310
Show file tree

Hide file tree

Showing 6 changed files with 122 additions and 10 deletions.
diff --git a/sandbox/SandboxConsoleApp/Program.cs b/sandbox/SandboxConsoleApp/Program.cs
@@ -25,6 +25,15 @@
 using System.Text;
 using System.Xml.Linq;
 
+var range = Enumerable.Range(1, 200).Select(x => (byte)x).ToArray();
+var hoge = new MemoryStream(range, 10, range.Length - 10, false, true);
+hoge.Position = 49;
+
+if (hoge.TryGetBuffer(out var buffer))
+{
+    Console.WriteLine(buffer);
+}
+
 
 Console.WriteLine("---");
 //var bin = MemoryPackSerializer.Serialize("hogehoge");

diff --git a/src/MemoryPack.Core/MemoryPackSerializer.Deserialize.cs b/src/MemoryPack.Core/MemoryPackSerializer.Deserialize.cs
@@ -18,7 +18,7 @@ public static partial class MemoryPackSerializer
         return value;
     }
 
-    public static void Deserialize<T>(ReadOnlySpan<byte> buffer, ref T? value, MemoryPackSerializerOptions? options = default)
+    public static int Deserialize<T>(ReadOnlySpan<byte> buffer, ref T? value, MemoryPackSerializerOptions? options = default)
     {
         if (!RuntimeHelpers.IsReferenceOrContainsReferences<T>())
         {
@@ -27,7 +27,7 @@ public static void Deserialize<T>(ReadOnlySpan<byte> buffer, ref T? value, Memor
                 MemoryPackSerializationException.ThrowInvalidRange(Unsafe.SizeOf<T>(), buffer.Length);
             }
             value = Unsafe.ReadUnaligned<T>(ref MemoryMarshal.GetReference(buffer));
-            return;
+            return Unsafe.SizeOf<T>();
         }
 
         var state = threadStaticReaderOptionalState;
@@ -41,6 +41,7 @@ public static void Deserialize<T>(ReadOnlySpan<byte> buffer, ref T? value, Memor
         try
         {
             reader.ReadValue(ref value);
+            return reader.Consumed;
         }
         finally
         {
@@ -56,7 +57,7 @@ public static void Deserialize<T>(ReadOnlySpan<byte> buffer, ref T? value, Memor
         return value;
     }
 
-    public static void Deserialize<T>(in ReadOnlySequence<byte> buffer, ref T? value, MemoryPackSerializerOptions? options = default)
+    public static int Deserialize<T>(in ReadOnlySequence<byte> buffer, ref T? value, MemoryPackSerializerOptions? options = default)
     {
         var state = threadStaticReaderOptionalState;
         if (state == null)
@@ -69,6 +70,7 @@ public static void Deserialize<T>(in ReadOnlySequence<byte> buffer, ref T? value
         try
         {
             reader.ReadValue(ref value);
+            return reader.Consumed;
         }
         finally
         {
@@ -79,6 +81,18 @@ public static void Deserialize<T>(in ReadOnlySequence<byte> buffer, ref T? value
 
     public static async ValueTask<T?> DeserializeAsync<T>(Stream stream, MemoryPackSerializerOptions? options = default, CancellationToken cancellationToken = default)
     {
+        if (stream is MemoryStream ms && ms.TryGetBuffer(out ArraySegment<byte> streamBuffer))
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+            T? value = default;
+            var bytesRead = Deserialize<T>(streamBuffer.AsSpan(checked((int)ms.Position)), ref value, options);
+
+            // Emulate that we had actually "read" from the stream.
+            ms.Seek(bytesRead, SeekOrigin.Current);
+
+            return value;
+        }
+
         var builder = ReusableReadOnlySequenceBuilderPool.Rent();
         try
         {

diff --git a/src/MemoryPack.Core/MemoryPackSerializer.NonGenerics.cs b/src/MemoryPack.Core/MemoryPackSerializer.NonGenerics.cs
@@ -91,7 +91,7 @@ public static async ValueTask SerializeAsync(Type type, Stream stream, object? v
         return value;
     }
 
-    public static void Deserialize(Type type, ReadOnlySpan<byte> buffer, ref object? value, MemoryPackSerializerOptions? options = default)
+    public static int Deserialize(Type type, ReadOnlySpan<byte> buffer, ref object? value, MemoryPackSerializerOptions? options = default)
     {
         var state = threadStaticReaderOptionalState;
         if (state == null)
@@ -104,6 +104,7 @@ public static void Deserialize(Type type, ReadOnlySpan<byte> buffer, ref object?
         try
         {
             reader.GetFormatter(type).Deserialize(ref reader, ref value);
+            return reader.Consumed;
         }
         finally
         {
@@ -119,7 +120,7 @@ public static void Deserialize(Type type, ReadOnlySpan<byte> buffer, ref object?
         return value;
     }
 
-    public static void Deserialize(Type type, in ReadOnlySequence<byte> buffer, ref object? value, MemoryPackSerializerOptions? options = default)
+    public static int Deserialize(Type type, in ReadOnlySequence<byte> buffer, ref object? value, MemoryPackSerializerOptions? options = default)
     {
         var state = threadStaticReaderOptionalState;
         if (state == null)
@@ -132,6 +133,7 @@ public static void Deserialize(Type type, in ReadOnlySequence<byte> buffer, ref
         try
         {
             reader.GetFormatter(type).Deserialize(ref reader, ref value);
+            return reader.Consumed;
         }
         finally
         {
@@ -142,6 +144,18 @@ public static void Deserialize(Type type, in ReadOnlySequence<byte> buffer, ref
 
     public static async ValueTask<object?> DeserializeAsync(Type type, Stream stream, MemoryPackSerializerOptions? options = default, CancellationToken cancellationToken = default)
     {
+        if (stream is MemoryStream ms && ms.TryGetBuffer(out ArraySegment<byte> streamBuffer))
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+            object? value = default;
+            var bytesRead = Deserialize(type, streamBuffer.AsSpan(checked((int)ms.Position)), ref value, options);
+
+            // Emulate that we had actually "read" from the stream.
+            ms.Seek(bytesRead, SeekOrigin.Current);
+
+            return value;
+        }
+
         var builder = ReusableReadOnlySequenceBuilderPool.Rent();
         try
         {

diff --git a/...ity/Assets/Plugins/MemoryPack/Runtime/MemoryPack.Core/MemoryPackSerializer.Deserialize.cs b/...ity/Assets/Plugins/MemoryPack/Runtime/MemoryPack.Core/MemoryPackSerializer.Deserialize.cs
@@ -27,7 +27,7 @@ public static partial class MemoryPackSerializer
         return value;
     }
 
-    public static void Deserialize<T>(ReadOnlySpan<byte> buffer, ref T? value, MemoryPackSerializerOptions? options = default)
+    public static int Deserialize<T>(ReadOnlySpan<byte> buffer, ref T? value, MemoryPackSerializerOptions? options = default)
     {
         if (!RuntimeHelpers.IsReferenceOrContainsReferences<T>())
         {
@@ -36,7 +36,7 @@ public static void Deserialize<T>(ReadOnlySpan<byte> buffer, ref T? value, Memor
                 MemoryPackSerializationException.ThrowInvalidRange(Unsafe.SizeOf<T>(), buffer.Length);
             }
             value = Unsafe.ReadUnaligned<T>(ref MemoryMarshal.GetReference(buffer));
-            return;
+            return Unsafe.SizeOf<T>();
         }
 
         var state = threadStaticReaderOptionalState;
@@ -50,6 +50,7 @@ public static void Deserialize<T>(ReadOnlySpan<byte> buffer, ref T? value, Memor
         try
         {
             reader.ReadValue(ref value);
+            return reader.Consumed;
         }
         finally
         {
@@ -65,7 +66,7 @@ public static void Deserialize<T>(ReadOnlySpan<byte> buffer, ref T? value, Memor
         return value;
     }
 
-    public static void Deserialize<T>(in ReadOnlySequence<byte> buffer, ref T? value, MemoryPackSerializerOptions? options = default)
+    public static int Deserialize<T>(in ReadOnlySequence<byte> buffer, ref T? value, MemoryPackSerializerOptions? options = default)
     {
         var state = threadStaticReaderOptionalState;
         if (state == null)
@@ -78,6 +79,7 @@ public static void Deserialize<T>(in ReadOnlySequence<byte> buffer, ref T? value
         try
         {
             reader.ReadValue(ref value);
+            return reader.Consumed;
         }
         finally
         {
@@ -88,6 +90,18 @@ public static void Deserialize<T>(in ReadOnlySequence<byte> buffer, ref T? value
 
     public static async ValueTask<T?> DeserializeAsync<T>(Stream stream, MemoryPackSerializerOptions? options = default, CancellationToken cancellationToken = default)
     {
+        if (stream is MemoryStream ms && ms.TryGetBuffer(out ArraySegment<byte> streamBuffer))
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+            T? value = default;
+            var bytesRead = Deserialize<T>(streamBuffer.AsSpan(checked((int)ms.Position)), ref value, options);
+
+            // Emulate that we had actually "read" from the stream.
+            ms.Seek(bytesRead, SeekOrigin.Current);
+
+            return value;
+        }
+
         var builder = ReusableReadOnlySequenceBuilderPool.Rent();
         try
         {

diff --git a/...ity/Assets/Plugins/MemoryPack/Runtime/MemoryPack.Core/MemoryPackSerializer.NonGenerics.cs b/...ity/Assets/Plugins/MemoryPack/Runtime/MemoryPack.Core/MemoryPackSerializer.NonGenerics.cs
@@ -100,7 +100,7 @@ public static async ValueTask SerializeAsync(Type type, Stream stream, object? v
         return value;
     }
 
-    public static void Deserialize(Type type, ReadOnlySpan<byte> buffer, ref object? value, MemoryPackSerializerOptions? options = default)
+    public static int Deserialize(Type type, ReadOnlySpan<byte> buffer, ref object? value, MemoryPackSerializerOptions? options = default)
     {
         var state = threadStaticReaderOptionalState;
         if (state == null)
@@ -113,6 +113,7 @@ public static void Deserialize(Type type, ReadOnlySpan<byte> buffer, ref object?
         try
         {
             reader.GetFormatter(type).Deserialize(ref reader, ref value);
+            return reader.Consumed;
         }
         finally
         {
@@ -128,7 +129,7 @@ public static void Deserialize(Type type, ReadOnlySpan<byte> buffer, ref object?
         return value;
     }
 
-    public static void Deserialize(Type type, in ReadOnlySequence<byte> buffer, ref object? value, MemoryPackSerializerOptions? options = default)
+    public static int Deserialize(Type type, in ReadOnlySequence<byte> buffer, ref object? value, MemoryPackSerializerOptions? options = default)
     {
         var state = threadStaticReaderOptionalState;
         if (state == null)
@@ -141,6 +142,7 @@ public static void Deserialize(Type type, in ReadOnlySequence<byte> buffer, ref
         try
         {
             reader.GetFormatter(type).Deserialize(ref reader, ref value);
+            return reader.Consumed;
         }
         finally
         {
@@ -151,6 +153,18 @@ public static void Deserialize(Type type, in ReadOnlySequence<byte> buffer, ref
 
     public static async ValueTask<object?> DeserializeAsync(Type type, Stream stream, MemoryPackSerializerOptions? options = default, CancellationToken cancellationToken = default)
     {
+        if (stream is MemoryStream ms && ms.TryGetBuffer(out ArraySegment<byte> streamBuffer))
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+            object? value = default;
+            var bytesRead = Deserialize(type, streamBuffer.AsSpan(checked((int)ms.Position)), ref value, options);
+
+            // Emulate that we had actually "read" from the stream.
+            ms.Seek(bytesRead, SeekOrigin.Current);
+
+            return value;
+        }
+
         var builder = ReusableReadOnlySequenceBuilderPool.Rent();
         try
         {

diff --git a/tests/MemoryPack.Tests/StreamOptimizeTest.cs b/tests/MemoryPack.Tests/StreamOptimizeTest.cs
@@ -0,0 +1,47 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace MemoryPack.Tests;
+
+public class StreamOptimizeTest
+{
+    [Fact]
+    public async Task MemoryStream()
+    {
+        var ms = new MemoryStream();
+        await MemoryPackSerializer.SerializeAsync(ms, new[] { 1, 2, 3 });
+        var offset = ms.Position;
+        await MemoryPackSerializer.SerializeAsync(ms, new[] { 10, 20, 30 });
+        await MemoryPackSerializer.SerializeAsync(ms, new[] { 40, 50, 60 });
+
+        ms.Position = offset;
+
+        var data1 = await MemoryPackSerializer.DeserializeAsync<int[]>(ms);
+        var data2 = await MemoryPackSerializer.DeserializeAsync<int[]>(ms);
+
+        data1.Should().Equal(10, 20, 30);
+        data2.Should().Equal(40, 50, 60);
+    }
+
+    [Fact]
+    public async Task MemoryStreamNoGenerics()
+    {
+        var ms = new MemoryStream();
+        await MemoryPackSerializer.SerializeAsync(ms, new[] { 1, 2, 3 });
+        var offset = ms.Position;
+        await MemoryPackSerializer.SerializeAsync(ms, new[] { 10, 20, 30 });
+        await MemoryPackSerializer.SerializeAsync(ms, new[] { 40, 50, 60 });
+
+        ms.Position = offset;
+
+        var data1 = (int[]?)await MemoryPackSerializer.DeserializeAsync(typeof(int[]), ms);
+        var data2 = (int[]?)await MemoryPackSerializer.DeserializeAsync(typeof(int[]), ms);
+
+        data1.Should().Equal(10, 20, 30);
+        data2.Should().Equal(40, 50, 60);
+    }
+}