From 957c310a3eb0b61bcfc54f7237d1bf77e44a4549 Mon Sep 17 00:00:00 2001 From: neuecc Date: Sun, 20 Nov 2022 00:41:56 +0900 Subject: [PATCH] Improve MemoryPackSerializer.Deserialize(Stream) performance --- sandbox/SandboxConsoleApp/Program.cs | 9 ++++ .../MemoryPackSerializer.Deserialize.cs | 20 ++++++-- .../MemoryPackSerializer.NonGenerics.cs | 18 ++++++- .../MemoryPackSerializer.Deserialize.cs | 20 ++++++-- .../MemoryPackSerializer.NonGenerics.cs | 18 ++++++- tests/MemoryPack.Tests/StreamOptimizeTest.cs | 47 +++++++++++++++++++ 6 files changed, 122 insertions(+), 10 deletions(-) create mode 100644 tests/MemoryPack.Tests/StreamOptimizeTest.cs diff --git a/sandbox/SandboxConsoleApp/Program.cs b/sandbox/SandboxConsoleApp/Program.cs index 2e06f54d..7ddb90ee 100644 --- a/sandbox/SandboxConsoleApp/Program.cs +++ b/sandbox/SandboxConsoleApp/Program.cs @@ -25,6 +25,15 @@ using System.Text; using System.Xml.Linq; +var range = Enumerable.Range(1, 200).Select(x => (byte)x).ToArray(); +var hoge = new MemoryStream(range, 10, range.Length - 10, false, true); +hoge.Position = 49; + +if (hoge.TryGetBuffer(out var buffer)) +{ + Console.WriteLine(buffer); +} + Console.WriteLine("---"); //var bin = MemoryPackSerializer.Serialize("hogehoge"); diff --git a/src/MemoryPack.Core/MemoryPackSerializer.Deserialize.cs b/src/MemoryPack.Core/MemoryPackSerializer.Deserialize.cs index e2e447a3..e3dae160 100644 --- a/src/MemoryPack.Core/MemoryPackSerializer.Deserialize.cs +++ b/src/MemoryPack.Core/MemoryPackSerializer.Deserialize.cs @@ -18,7 +18,7 @@ public static partial class MemoryPackSerializer return value; } - public static void Deserialize(ReadOnlySpan buffer, ref T? value, MemoryPackSerializerOptions? options = default) + public static int Deserialize(ReadOnlySpan buffer, ref T? value, MemoryPackSerializerOptions? options = default) { if (!RuntimeHelpers.IsReferenceOrContainsReferences()) { @@ -27,7 +27,7 @@ public static void Deserialize(ReadOnlySpan buffer, ref T? value, Memor MemoryPackSerializationException.ThrowInvalidRange(Unsafe.SizeOf(), buffer.Length); } value = Unsafe.ReadUnaligned(ref MemoryMarshal.GetReference(buffer)); - return; + return Unsafe.SizeOf(); } var state = threadStaticReaderOptionalState; @@ -41,6 +41,7 @@ public static void Deserialize(ReadOnlySpan buffer, ref T? value, Memor try { reader.ReadValue(ref value); + return reader.Consumed; } finally { @@ -56,7 +57,7 @@ public static void Deserialize(ReadOnlySpan buffer, ref T? value, Memor return value; } - public static void Deserialize(in ReadOnlySequence buffer, ref T? value, MemoryPackSerializerOptions? options = default) + public static int Deserialize(in ReadOnlySequence buffer, ref T? value, MemoryPackSerializerOptions? options = default) { var state = threadStaticReaderOptionalState; if (state == null) @@ -69,6 +70,7 @@ public static void Deserialize(in ReadOnlySequence buffer, ref T? value try { reader.ReadValue(ref value); + return reader.Consumed; } finally { @@ -79,6 +81,18 @@ public static void Deserialize(in ReadOnlySequence buffer, ref T? value public static async ValueTask DeserializeAsync(Stream stream, MemoryPackSerializerOptions? options = default, CancellationToken cancellationToken = default) { + if (stream is MemoryStream ms && ms.TryGetBuffer(out ArraySegment streamBuffer)) + { + cancellationToken.ThrowIfCancellationRequested(); + T? value = default; + var bytesRead = Deserialize(streamBuffer.AsSpan(checked((int)ms.Position)), ref value, options); + + // Emulate that we had actually "read" from the stream. + ms.Seek(bytesRead, SeekOrigin.Current); + + return value; + } + var builder = ReusableReadOnlySequenceBuilderPool.Rent(); try { diff --git a/src/MemoryPack.Core/MemoryPackSerializer.NonGenerics.cs b/src/MemoryPack.Core/MemoryPackSerializer.NonGenerics.cs index 6e65e157..29ee50ef 100644 --- a/src/MemoryPack.Core/MemoryPackSerializer.NonGenerics.cs +++ b/src/MemoryPack.Core/MemoryPackSerializer.NonGenerics.cs @@ -91,7 +91,7 @@ public static async ValueTask SerializeAsync(Type type, Stream stream, object? v return value; } - public static void Deserialize(Type type, ReadOnlySpan buffer, ref object? value, MemoryPackSerializerOptions? options = default) + public static int Deserialize(Type type, ReadOnlySpan buffer, ref object? value, MemoryPackSerializerOptions? options = default) { var state = threadStaticReaderOptionalState; if (state == null) @@ -104,6 +104,7 @@ public static void Deserialize(Type type, ReadOnlySpan buffer, ref object? try { reader.GetFormatter(type).Deserialize(ref reader, ref value); + return reader.Consumed; } finally { @@ -119,7 +120,7 @@ public static void Deserialize(Type type, ReadOnlySpan buffer, ref object? return value; } - public static void Deserialize(Type type, in ReadOnlySequence buffer, ref object? value, MemoryPackSerializerOptions? options = default) + public static int Deserialize(Type type, in ReadOnlySequence buffer, ref object? value, MemoryPackSerializerOptions? options = default) { var state = threadStaticReaderOptionalState; if (state == null) @@ -132,6 +133,7 @@ public static void Deserialize(Type type, in ReadOnlySequence buffer, ref try { reader.GetFormatter(type).Deserialize(ref reader, ref value); + return reader.Consumed; } finally { @@ -142,6 +144,18 @@ public static void Deserialize(Type type, in ReadOnlySequence buffer, ref public static async ValueTask DeserializeAsync(Type type, Stream stream, MemoryPackSerializerOptions? options = default, CancellationToken cancellationToken = default) { + if (stream is MemoryStream ms && ms.TryGetBuffer(out ArraySegment streamBuffer)) + { + cancellationToken.ThrowIfCancellationRequested(); + object? value = default; + var bytesRead = Deserialize(type, streamBuffer.AsSpan(checked((int)ms.Position)), ref value, options); + + // Emulate that we had actually "read" from the stream. + ms.Seek(bytesRead, SeekOrigin.Current); + + return value; + } + var builder = ReusableReadOnlySequenceBuilderPool.Rent(); try { diff --git a/src/MemoryPack.Unity/Assets/Plugins/MemoryPack/Runtime/MemoryPack.Core/MemoryPackSerializer.Deserialize.cs b/src/MemoryPack.Unity/Assets/Plugins/MemoryPack/Runtime/MemoryPack.Core/MemoryPackSerializer.Deserialize.cs index 3a5c510a..64d9f81c 100644 --- a/src/MemoryPack.Unity/Assets/Plugins/MemoryPack/Runtime/MemoryPack.Core/MemoryPackSerializer.Deserialize.cs +++ b/src/MemoryPack.Unity/Assets/Plugins/MemoryPack/Runtime/MemoryPack.Core/MemoryPackSerializer.Deserialize.cs @@ -27,7 +27,7 @@ public static partial class MemoryPackSerializer return value; } - public static void Deserialize(ReadOnlySpan buffer, ref T? value, MemoryPackSerializerOptions? options = default) + public static int Deserialize(ReadOnlySpan buffer, ref T? value, MemoryPackSerializerOptions? options = default) { if (!RuntimeHelpers.IsReferenceOrContainsReferences()) { @@ -36,7 +36,7 @@ public static void Deserialize(ReadOnlySpan buffer, ref T? value, Memor MemoryPackSerializationException.ThrowInvalidRange(Unsafe.SizeOf(), buffer.Length); } value = Unsafe.ReadUnaligned(ref MemoryMarshal.GetReference(buffer)); - return; + return Unsafe.SizeOf(); } var state = threadStaticReaderOptionalState; @@ -50,6 +50,7 @@ public static void Deserialize(ReadOnlySpan buffer, ref T? value, Memor try { reader.ReadValue(ref value); + return reader.Consumed; } finally { @@ -65,7 +66,7 @@ public static void Deserialize(ReadOnlySpan buffer, ref T? value, Memor return value; } - public static void Deserialize(in ReadOnlySequence buffer, ref T? value, MemoryPackSerializerOptions? options = default) + public static int Deserialize(in ReadOnlySequence buffer, ref T? value, MemoryPackSerializerOptions? options = default) { var state = threadStaticReaderOptionalState; if (state == null) @@ -78,6 +79,7 @@ public static void Deserialize(in ReadOnlySequence buffer, ref T? value try { reader.ReadValue(ref value); + return reader.Consumed; } finally { @@ -88,6 +90,18 @@ public static void Deserialize(in ReadOnlySequence buffer, ref T? value public static async ValueTask DeserializeAsync(Stream stream, MemoryPackSerializerOptions? options = default, CancellationToken cancellationToken = default) { + if (stream is MemoryStream ms && ms.TryGetBuffer(out ArraySegment streamBuffer)) + { + cancellationToken.ThrowIfCancellationRequested(); + T? value = default; + var bytesRead = Deserialize(streamBuffer.AsSpan(checked((int)ms.Position)), ref value, options); + + // Emulate that we had actually "read" from the stream. + ms.Seek(bytesRead, SeekOrigin.Current); + + return value; + } + var builder = ReusableReadOnlySequenceBuilderPool.Rent(); try { diff --git a/src/MemoryPack.Unity/Assets/Plugins/MemoryPack/Runtime/MemoryPack.Core/MemoryPackSerializer.NonGenerics.cs b/src/MemoryPack.Unity/Assets/Plugins/MemoryPack/Runtime/MemoryPack.Core/MemoryPackSerializer.NonGenerics.cs index ce83ea90..27026aa1 100644 --- a/src/MemoryPack.Unity/Assets/Plugins/MemoryPack/Runtime/MemoryPack.Core/MemoryPackSerializer.NonGenerics.cs +++ b/src/MemoryPack.Unity/Assets/Plugins/MemoryPack/Runtime/MemoryPack.Core/MemoryPackSerializer.NonGenerics.cs @@ -100,7 +100,7 @@ public static async ValueTask SerializeAsync(Type type, Stream stream, object? v return value; } - public static void Deserialize(Type type, ReadOnlySpan buffer, ref object? value, MemoryPackSerializerOptions? options = default) + public static int Deserialize(Type type, ReadOnlySpan buffer, ref object? value, MemoryPackSerializerOptions? options = default) { var state = threadStaticReaderOptionalState; if (state == null) @@ -113,6 +113,7 @@ public static void Deserialize(Type type, ReadOnlySpan buffer, ref object? try { reader.GetFormatter(type).Deserialize(ref reader, ref value); + return reader.Consumed; } finally { @@ -128,7 +129,7 @@ public static void Deserialize(Type type, ReadOnlySpan buffer, ref object? return value; } - public static void Deserialize(Type type, in ReadOnlySequence buffer, ref object? value, MemoryPackSerializerOptions? options = default) + public static int Deserialize(Type type, in ReadOnlySequence buffer, ref object? value, MemoryPackSerializerOptions? options = default) { var state = threadStaticReaderOptionalState; if (state == null) @@ -141,6 +142,7 @@ public static void Deserialize(Type type, in ReadOnlySequence buffer, ref try { reader.GetFormatter(type).Deserialize(ref reader, ref value); + return reader.Consumed; } finally { @@ -151,6 +153,18 @@ public static void Deserialize(Type type, in ReadOnlySequence buffer, ref public static async ValueTask DeserializeAsync(Type type, Stream stream, MemoryPackSerializerOptions? options = default, CancellationToken cancellationToken = default) { + if (stream is MemoryStream ms && ms.TryGetBuffer(out ArraySegment streamBuffer)) + { + cancellationToken.ThrowIfCancellationRequested(); + object? value = default; + var bytesRead = Deserialize(type, streamBuffer.AsSpan(checked((int)ms.Position)), ref value, options); + + // Emulate that we had actually "read" from the stream. + ms.Seek(bytesRead, SeekOrigin.Current); + + return value; + } + var builder = ReusableReadOnlySequenceBuilderPool.Rent(); try { diff --git a/tests/MemoryPack.Tests/StreamOptimizeTest.cs b/tests/MemoryPack.Tests/StreamOptimizeTest.cs new file mode 100644 index 00000000..6ecb0a76 --- /dev/null +++ b/tests/MemoryPack.Tests/StreamOptimizeTest.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace MemoryPack.Tests; + +public class StreamOptimizeTest +{ + [Fact] + public async Task MemoryStream() + { + var ms = new MemoryStream(); + await MemoryPackSerializer.SerializeAsync(ms, new[] { 1, 2, 3 }); + var offset = ms.Position; + await MemoryPackSerializer.SerializeAsync(ms, new[] { 10, 20, 30 }); + await MemoryPackSerializer.SerializeAsync(ms, new[] { 40, 50, 60 }); + + ms.Position = offset; + + var data1 = await MemoryPackSerializer.DeserializeAsync(ms); + var data2 = await MemoryPackSerializer.DeserializeAsync(ms); + + data1.Should().Equal(10, 20, 30); + data2.Should().Equal(40, 50, 60); + } + + [Fact] + public async Task MemoryStreamNoGenerics() + { + var ms = new MemoryStream(); + await MemoryPackSerializer.SerializeAsync(ms, new[] { 1, 2, 3 }); + var offset = ms.Position; + await MemoryPackSerializer.SerializeAsync(ms, new[] { 10, 20, 30 }); + await MemoryPackSerializer.SerializeAsync(ms, new[] { 40, 50, 60 }); + + ms.Position = offset; + + var data1 = (int[]?)await MemoryPackSerializer.DeserializeAsync(typeof(int[]), ms); + var data2 = (int[]?)await MemoryPackSerializer.DeserializeAsync(typeof(int[]), ms); + + data1.Should().Equal(10, 20, 30); + data2.Should().Equal(40, 50, 60); + } +}