From cb407b98b74d3e8382ee81a3bdf61b07f129d542 Mon Sep 17 00:00:00 2001 From: Brennan Date: Wed, 15 Nov 2023 11:00:19 -0800 Subject: [PATCH 01/14] Add PipeWriter overloads to Json --- .../System.Text.Json/ref/System.Text.Json.cs | 4 + .../ref/System.Text.Json.csproj | 1 + .../src/System.Text.Json.csproj | 1 + .../JsonSerializer.Write.Stream.cs | 55 +++++++ .../Serialization/Metadata/JsonTypeInfo.cs | 1 + .../Metadata/JsonTypeInfoOfT.WriteHelpers.cs | 138 ++++++++++++++++++ 6 files changed, 200 insertions(+) diff --git a/src/libraries/System.Text.Json/ref/System.Text.Json.cs b/src/libraries/System.Text.Json/ref/System.Text.Json.cs index 25ccb474dbbb6..6ac412bf120b9 100644 --- a/src/libraries/System.Text.Json/ref/System.Text.Json.cs +++ b/src/libraries/System.Text.Json/ref/System.Text.Json.cs @@ -307,8 +307,12 @@ public static void Serialize(System.Text.Json.Utf8JsonWriter writer, object? val public static System.Threading.Tasks.Task SerializeAsync(System.IO.Stream utf8Json, object? value, System.Type inputType, System.Text.Json.Serialization.JsonSerializerContext context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } [System.Diagnostics.CodeAnalysis.RequiresDynamicCodeAttribute("JSON serialization and deserialization might require types that cannot be statically analyzed and might need runtime code generation. Use System.Text.Json source generation for native AOT applications.")] [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("JSON serialization and deserialization might require types that cannot be statically analyzed. Use the overload that takes a JsonTypeInfo or JsonSerializerContext, or make sure all of the required types are preserved.")] + public static System.Threading.Tasks.Task SerializeAsync(System.IO.Pipelines.PipeWriter utf8Json, TValue value, System.Text.Json.JsonSerializerOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + [System.Diagnostics.CodeAnalysis.RequiresDynamicCodeAttribute("JSON serialization and deserialization might require types that cannot be statically analyzed and might need runtime code generation. Use System.Text.Json source generation for native AOT applications.")] + [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("JSON serialization and deserialization might require types that cannot be statically analyzed. Use the overload that takes a JsonTypeInfo or JsonSerializerContext, or make sure all of the required types are preserved.")] public static System.Threading.Tasks.Task SerializeAsync(System.IO.Stream utf8Json, TValue value, System.Text.Json.JsonSerializerOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public static System.Threading.Tasks.Task SerializeAsync(System.IO.Stream utf8Json, TValue value, System.Text.Json.Serialization.Metadata.JsonTypeInfo jsonTypeInfo, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public static System.Threading.Tasks.Task SerializeAsync(System.IO.Pipelines.PipeWriter utf8Json, TValue value, System.Text.Json.Serialization.Metadata.JsonTypeInfo jsonTypeInfo, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public static System.Text.Json.JsonDocument SerializeToDocument(object? value, System.Text.Json.Serialization.Metadata.JsonTypeInfo jsonTypeInfo) { throw null; } [System.Diagnostics.CodeAnalysis.RequiresDynamicCodeAttribute("JSON serialization and deserialization might require types that cannot be statically analyzed and might need runtime code generation. Use System.Text.Json source generation for native AOT applications.")] [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("JSON serialization and deserialization might require types that cannot be statically analyzed. Use the overload that takes a JsonTypeInfo or JsonSerializerContext, or make sure all of the required types are preserved.")] diff --git a/src/libraries/System.Text.Json/ref/System.Text.Json.csproj b/src/libraries/System.Text.Json/ref/System.Text.Json.csproj index 48187a11e58ea..9e41bc3097af0 100644 --- a/src/libraries/System.Text.Json/ref/System.Text.Json.csproj +++ b/src/libraries/System.Text.Json/ref/System.Text.Json.csproj @@ -26,6 +26,7 @@ + diff --git a/src/libraries/System.Text.Json/src/System.Text.Json.csproj b/src/libraries/System.Text.Json/src/System.Text.Json.csproj index c95276f2db2bc..57f25a2bfb0c2 100644 --- a/src/libraries/System.Text.Json/src/System.Text.Json.csproj +++ b/src/libraries/System.Text.Json/src/System.Text.Json.csproj @@ -372,6 +372,7 @@ The System.Text.Json library is built-in as part of the shared framework in .NET + diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs index 9c678e37ad394..031fd14508648 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs @@ -3,6 +3,7 @@ using System.Diagnostics.CodeAnalysis; using System.IO; +using System.IO.Pipelines; using System.Text.Json.Serialization; using System.Text.Json.Serialization.Metadata; using System.Threading; @@ -53,6 +54,32 @@ public static Task SerializeAsync( return jsonTypeInfo.SerializeAsync(utf8Json, value, cancellationToken); } + /// + /// g + /// + /// + /// + /// + /// + /// + /// + [RequiresUnreferencedCode(SerializationUnreferencedCodeMessage)] + [RequiresDynamicCode(SerializationRequiresDynamicCodeMessage)] + public static Task SerializeAsync( + System.IO.Pipelines.PipeWriter utf8Json, + TValue value, + JsonSerializerOptions? options = null, + CancellationToken cancellationToken = default) + { + if (utf8Json is null) + { + ThrowHelper.ThrowArgumentNullException(nameof(utf8Json)); + } + + JsonTypeInfo jsonTypeInfo = GetTypeInfo(options); + return jsonTypeInfo.SerializeAsync(utf8Json, value, cancellationToken); + } + /// /// Converts the provided value to UTF-8 encoded JSON text and write it to the . /// @@ -187,6 +214,34 @@ public static Task SerializeAsync( return jsonTypeInfo.SerializeAsync(utf8Json, value, cancellationToken); } + /// + /// tmp + /// + /// + /// + /// + /// + /// + /// + public static Task SerializeAsync( + PipeWriter utf8Json, + TValue value, + JsonTypeInfo jsonTypeInfo, + CancellationToken cancellationToken = default) + { + if (utf8Json is null) + { + ThrowHelper.ThrowArgumentNullException(nameof(utf8Json)); + } + if (jsonTypeInfo is null) + { + ThrowHelper.ThrowArgumentNullException(nameof(jsonTypeInfo)); + } + + jsonTypeInfo.EnsureConfigured(); + return jsonTypeInfo.SerializeAsync(utf8Json, value, cancellationToken); + } + /// /// Converts the provided value to UTF-8 encoded JSON text and write it to the . /// diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfo.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfo.cs index e915c5bf4879e..9afcc553b9ede 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfo.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfo.cs @@ -990,6 +990,7 @@ public JsonPropertyInfo CreateJsonPropertyInfo(Type propertyType, string name) // Untyped, root-level serialization methods internal abstract void SerializeAsObject(Utf8JsonWriter writer, object? rootValue); internal abstract Task SerializeAsObjectAsync(Stream utf8Json, object? rootValue, CancellationToken cancellationToken); + internal abstract Task SerializeAsObjectAsync(System.IO.Pipelines.PipeWriter utf8Json, object? rootValue, CancellationToken cancellationToken); internal abstract void SerializeAsObject(Stream utf8Json, object? rootValue); // Untyped, root-level deserialization methods diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs index d916334c1925a..b94aa0966489c 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.IO; +using System.IO.Pipelines; using System.Text.Json.Serialization.Converters; using System.Threading; using System.Threading.Tasks; @@ -188,6 +189,140 @@ rootValue is not null && } } + internal async Task SerializeAsync( + PipeWriter utf8Json, + T? rootValue, + CancellationToken cancellationToken, + object? rootValueBoxed = null) + { + Debug.Assert(IsConfigured); + Debug.Assert(rootValueBoxed is null || rootValueBoxed is T); + + if (CanUseSerializeHandlerInStreaming) + { + // Short-circuit calls into SerializeHandler, if the `CanUseSerializeHandlerInStreaming` heuristic allows it. + //throw new NotImplementedException(); + Debug.Assert(SerializeHandler != null); + //Debug.Assert(CanUseSerializeHandler); + //Debug.Assert(Converter is JsonMetadataServicesConverter); + + //using var bufferWriter = new PooledByteBufferWriter(Options.DefaultBufferSize); + //Utf8JsonWriter writer = Utf8JsonWriterCache.RentWriter(Options, bufferWriter); + Utf8JsonWriter writer = Utf8JsonWriterCache.RentWriterAndBuffer(Options, out PooledByteBufferWriter bufferWriter); + writer.Reset(utf8Json); + + try + { + SerializeHandler(writer, rootValue!); + writer.Flush(); + } + finally + { + // Record the serialization size in both successful and failed operations, + // since we want to immediately opt out of the fast path if it exceeds the threshold. + OnRootLevelAsyncSerializationCompleted(writer.BytesCommitted + writer.BytesPending); + + Utf8JsonWriterCache.ReturnWriterAndBuffer(writer, bufferWriter); + } + + //await bufferWriter.WriteToStreamAsync(utf8Json, cancellationToken).ConfigureAwait(false); + await utf8Json.FlushAsync(cancellationToken).ConfigureAwait(false); + } + else if ( +#if NETCOREAPP + !typeof(T).IsValueType && +#endif + Converter.CanBePolymorphic && + rootValue is not null && + Options.TryGetPolymorphicTypeInfoForRootType(rootValue, out JsonTypeInfo? derivedTypeInfo)) + { + Debug.Assert(typeof(T) == typeof(object)); + await derivedTypeInfo.SerializeAsObjectAsync(utf8Json, rootValue, cancellationToken).ConfigureAwait(false); + } + else + { + bool isFinalBlock; + WriteStack state = default; + state.Initialize(this, + rootValueBoxed, + supportContinuation: true, + supportAsync: true); + + state.CancellationToken = cancellationToken; + + //using var bufferWriter = new PooledByteBufferWriter(Options.DefaultBufferSize); + using var writer = new Utf8JsonWriter(utf8Json, Options.GetWriterOptions()); + + try + { + do + { + state.FlushThreshold = 4096;//(int)(bufferWriter.Capacity * JsonSerializer.FlushThreshold); + + try + { + isFinalBlock = EffectiveConverter.WriteCore(writer, rootValue, Options, ref state); + writer.Flush(); + + if (state.SuppressFlush) + { + Debug.Assert(!isFinalBlock); + Debug.Assert(state.PendingTask is not null); + state.SuppressFlush = false; + } + else + { + //await bufferWriter.WriteToStreamAsync(utf8Json, cancellationToken).ConfigureAwait(false); + //bufferWriter.Clear(); + await utf8Json.FlushAsync(cancellationToken).ConfigureAwait(false); + } + } + finally + { + // Await any pending resumable converter tasks (currently these can only be IAsyncEnumerator.MoveNextAsync() tasks). + // Note that pending tasks are always awaited, even if an exception has been thrown or the cancellation token has fired. + if (state.PendingTask is not null) + { + // Exceptions should only be propagated by the resuming converter +#if NET8_0_OR_GREATER + await state.PendingTask.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); +#else + try + { + await state.PendingTask.ConfigureAwait(false); + } + catch { } +#endif + } + + // Dispose any pending async disposables (currently these can only be completed IAsyncEnumerators). + if (state.CompletedAsyncDisposables?.Count > 0) + { + await state.DisposeCompletedAsyncDisposables().ConfigureAwait(false); + } + } + + } while (!isFinalBlock); + } + catch + { + // On exception, walk the WriteStack for any orphaned disposables and try to dispose them. + await state.DisposePendingDisposablesOnExceptionAsync().ConfigureAwait(false); + throw; + } + + if (CanUseSerializeHandler) + { + // On successful serialization, record the serialization size + // to determine potential suitability of the type for + // fast-path serialization in streaming methods. + Debug.Assert(writer.BytesPending == 0); + OnRootLevelAsyncSerializationCompleted(writer.BytesCommitted); + } + } + } + + // Root serialization method for non-async streaming serialization internal void Serialize( Stream utf8Json, @@ -274,6 +409,9 @@ internal sealed override void SerializeAsObject(Utf8JsonWriter writer, object? r internal sealed override Task SerializeAsObjectAsync(Stream utf8Json, object? rootValue, CancellationToken cancellationToken) => SerializeAsync(utf8Json, JsonSerializer.UnboxOnWrite(rootValue), cancellationToken, rootValue); + internal sealed override Task SerializeAsObjectAsync(System.IO.Pipelines.PipeWriter utf8Json, object? rootValue, CancellationToken cancellationToken) + => SerializeAsync(utf8Json, JsonSerializer.UnboxOnWrite(rootValue), cancellationToken, rootValue); + internal sealed override void SerializeAsObject(Stream utf8Json, object? rootValue) => Serialize(utf8Json, JsonSerializer.UnboxOnWrite(rootValue), rootValue); From 7512f264590d9777eaa917b386a7ad9f268fe2e4 Mon Sep 17 00:00:00 2001 From: Brennan Date: Sun, 3 Mar 2024 19:22:23 -0800 Subject: [PATCH 02/14] test --- src/libraries/NetCoreAppLibrary.props | 2 +- .../ref/System.IO.Pipelines.csproj | 5 + .../src/System.IO.Pipelines.csproj | 9 + .../System.Text.Json/ref/System.Text.Json.cs | 5 + .../JsonSerializer.Write.Stream.cs | 91 ++++++- .../Metadata/JsonTypeInfoOfT.WriteHelpers.cs | 15 +- .../Text/Json/Writer/Utf8JsonWriterCache.cs | 3 +- .../System.Text.Json.Tests/JsonTestHelper.cs | 256 ++++++++++++++++++ .../JsonSerializerApiValidation.cs | 5 + .../JsonSerializerWrapper.Reflection.cs | 74 +++++ .../MetadataTests/MetadataTests.cs | 5 + .../Serialization/NumberHandlingTests.cs | 5 + .../Serialization/PolymorphicTests.cs | 5 + .../Serialization/RequiredKeywordTests.cs | 5 + .../Utf8JsonReaderTests.cs | 76 +++--- 15 files changed, 511 insertions(+), 50 deletions(-) diff --git a/src/libraries/NetCoreAppLibrary.props b/src/libraries/NetCoreAppLibrary.props index 49fc1048b2b1c..2b0f1a0f2e51d 100644 --- a/src/libraries/NetCoreAppLibrary.props +++ b/src/libraries/NetCoreAppLibrary.props @@ -77,6 +77,7 @@ System.IO.MemoryMappedFiles; System.IO.Pipes; System.IO.Pipes.AccessControl; + System.IO.Pipelines; System.IO.UnmanagedMemoryStream; System.Linq; System.Linq.Expressions; @@ -220,7 +221,6 @@ Microsoft.Extensions.Options.DataAnnotations; Microsoft.Extensions.Primitives; System.Diagnostics.EventLog; - System.IO.Pipelines; System.Security.Cryptography.Xml; System.Threading.RateLimiting; diff --git a/src/libraries/System.IO.Pipelines/ref/System.IO.Pipelines.csproj b/src/libraries/System.IO.Pipelines/ref/System.IO.Pipelines.csproj index c87749fe98545..7148be9f761d9 100644 --- a/src/libraries/System.IO.Pipelines/ref/System.IO.Pipelines.csproj +++ b/src/libraries/System.IO.Pipelines/ref/System.IO.Pipelines.csproj @@ -7,6 +7,11 @@ + + + + + diff --git a/src/libraries/System.IO.Pipelines/src/System.IO.Pipelines.csproj b/src/libraries/System.IO.Pipelines/src/System.IO.Pipelines.csproj index c672633ad1681..c505f82d7e45f 100644 --- a/src/libraries/System.IO.Pipelines/src/System.IO.Pipelines.csproj +++ b/src/libraries/System.IO.Pipelines/src/System.IO.Pipelines.csproj @@ -65,4 +65,13 @@ System.IO.Pipelines.PipeReader + + + + + + + + + diff --git a/src/libraries/System.Text.Json/ref/System.Text.Json.cs b/src/libraries/System.Text.Json/ref/System.Text.Json.cs index 6ac412bf120b9..ef411f099038e 100644 --- a/src/libraries/System.Text.Json/ref/System.Text.Json.cs +++ b/src/libraries/System.Text.Json/ref/System.Text.Json.cs @@ -313,6 +313,11 @@ public static void Serialize(System.Text.Json.Utf8JsonWriter writer, object? val public static System.Threading.Tasks.Task SerializeAsync(System.IO.Stream utf8Json, TValue value, System.Text.Json.JsonSerializerOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public static System.Threading.Tasks.Task SerializeAsync(System.IO.Stream utf8Json, TValue value, System.Text.Json.Serialization.Metadata.JsonTypeInfo jsonTypeInfo, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public static System.Threading.Tasks.Task SerializeAsync(System.IO.Pipelines.PipeWriter utf8Json, TValue value, System.Text.Json.Serialization.Metadata.JsonTypeInfo jsonTypeInfo, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public static System.Threading.Tasks.Task SerializeAsync(System.IO.Pipelines.PipeWriter utf8Json, object? value, System.Text.Json.Serialization.Metadata.JsonTypeInfo jsonTypeInfo, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + [System.Diagnostics.CodeAnalysis.RequiresDynamicCodeAttribute("JSON serialization and deserialization might require types that cannot be statically analyzed and might need runtime code generation. Use System.Text.Json source generation for native AOT applications.")] + [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("JSON serialization and deserialization might require types that cannot be statically analyzed. Use the overload that takes a JsonTypeInfo or JsonSerializerContext, or make sure all of the required types are preserved.")] + public static System.Threading.Tasks.Task SerializeAsync(System.IO.Pipelines.PipeWriter utf8Json, object? value, System.Type inputType, System.Text.Json.JsonSerializerOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public static System.Threading.Tasks.Task SerializeAsync(System.IO.Pipelines.PipeWriter utf8Json, object? value, System.Type inputType, System.Text.Json.Serialization.JsonSerializerContext context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public static System.Text.Json.JsonDocument SerializeToDocument(object? value, System.Text.Json.Serialization.Metadata.JsonTypeInfo jsonTypeInfo) { throw null; } [System.Diagnostics.CodeAnalysis.RequiresDynamicCodeAttribute("JSON serialization and deserialization might require types that cannot be statically analyzed and might need runtime code generation. Use System.Text.Json source generation for native AOT applications.")] [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("JSON serialization and deserialization might require types that cannot be statically analyzed. Use the overload that takes a JsonTypeInfo or JsonSerializerContext, or make sure all of the required types are preserved.")] diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs index 031fd14508648..0a270fb9be8fc 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs @@ -66,7 +66,7 @@ public static Task SerializeAsync( [RequiresUnreferencedCode(SerializationUnreferencedCodeMessage)] [RequiresDynamicCode(SerializationRequiresDynamicCodeMessage)] public static Task SerializeAsync( - System.IO.Pipelines.PipeWriter utf8Json, + PipeWriter utf8Json, TValue value, JsonSerializerOptions? options = null, CancellationToken cancellationToken = default) @@ -80,6 +80,95 @@ public static Task SerializeAsync( return jsonTypeInfo.SerializeAsync(utf8Json, value, cancellationToken); } + /// + /// f + /// + /// + /// + /// + /// + /// + public static Task SerializeAsync( + PipeWriter utf8Json, + object? value, + JsonTypeInfo jsonTypeInfo, + CancellationToken cancellationToken = default) + { + if (utf8Json is null) + { + ThrowHelper.ThrowArgumentNullException(nameof(utf8Json)); + } + + if (jsonTypeInfo is null) + { + ThrowHelper.ThrowArgumentNullException(nameof(jsonTypeInfo)); + } + + jsonTypeInfo.EnsureConfigured(); + return jsonTypeInfo.SerializeAsObjectAsync(utf8Json, value, cancellationToken); + } + + /// + /// g + /// + /// + /// + /// + /// + /// + /// + public static Task SerializeAsync( + PipeWriter utf8Json, + object? value, + Type inputType, + JsonSerializerContext context, + CancellationToken cancellationToken = default) + { + if (utf8Json is null) + { + ThrowHelper.ThrowArgumentNullException(nameof(utf8Json)); + } + + if (context is null) + { + ThrowHelper.ThrowArgumentNullException(nameof(context)); + } + + ValidateInputType(value, inputType); + JsonTypeInfo jsonTypeInfo = GetTypeInfo(context, inputType); + + return jsonTypeInfo.SerializeAsObjectAsync(utf8Json, value, cancellationToken); + } + + /// + /// g + /// + /// + /// + /// + /// + /// + /// + [RequiresUnreferencedCode(SerializationUnreferencedCodeMessage)] + [RequiresDynamicCode(SerializationRequiresDynamicCodeMessage)] + public static Task SerializeAsync( + PipeWriter utf8Json, + object? value, + Type inputType, + JsonSerializerOptions? options = null, + CancellationToken cancellationToken = default) + { + if (utf8Json is null) + { + ThrowHelper.ThrowArgumentNullException(nameof(utf8Json)); + } + + ValidateInputType(value, inputType); + JsonTypeInfo jsonTypeInfo = GetTypeInfo(options, inputType); + + return jsonTypeInfo.SerializeAsObjectAsync(utf8Json, value, cancellationToken); + } + /// /// Converts the provided value to UTF-8 encoded JSON text and write it to the . /// diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs index b94aa0966489c..f9ea92bd8fa25 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs @@ -201,15 +201,11 @@ internal async Task SerializeAsync( if (CanUseSerializeHandlerInStreaming) { // Short-circuit calls into SerializeHandler, if the `CanUseSerializeHandlerInStreaming` heuristic allows it. - //throw new NotImplementedException(); Debug.Assert(SerializeHandler != null); - //Debug.Assert(CanUseSerializeHandler); - //Debug.Assert(Converter is JsonMetadataServicesConverter); + Debug.Assert(CanUseSerializeHandler); + Debug.Assert(Converter is JsonMetadataServicesConverter); - //using var bufferWriter = new PooledByteBufferWriter(Options.DefaultBufferSize); - //Utf8JsonWriter writer = Utf8JsonWriterCache.RentWriter(Options, bufferWriter); - Utf8JsonWriter writer = Utf8JsonWriterCache.RentWriterAndBuffer(Options, out PooledByteBufferWriter bufferWriter); - writer.Reset(utf8Json); + Utf8JsonWriter writer = Utf8JsonWriterCache.RentWriter(Options, utf8Json); try { @@ -222,10 +218,9 @@ internal async Task SerializeAsync( // since we want to immediately opt out of the fast path if it exceeds the threshold. OnRootLevelAsyncSerializationCompleted(writer.BytesCommitted + writer.BytesPending); - Utf8JsonWriterCache.ReturnWriterAndBuffer(writer, bufferWriter); + Utf8JsonWriterCache.ReturnWriter(writer); } - //await bufferWriter.WriteToStreamAsync(utf8Json, cancellationToken).ConfigureAwait(false); await utf8Json.FlushAsync(cancellationToken).ConfigureAwait(false); } else if ( @@ -272,8 +267,6 @@ rootValue is not null && } else { - //await bufferWriter.WriteToStreamAsync(utf8Json, cancellationToken).ConfigureAwait(false); - //bufferWriter.Clear(); await utf8Json.FlushAsync(cancellationToken).ConfigureAwait(false); } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriterCache.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriterCache.cs index b0d9528031286..bfe92487f01c8 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriterCache.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriterCache.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Buffers; using System.Diagnostics; namespace System.Text.Json @@ -40,7 +41,7 @@ public static Utf8JsonWriter RentWriterAndBuffer(JsonWriterOptions options, int return writer; } - public static Utf8JsonWriter RentWriter(JsonSerializerOptions options, PooledByteBufferWriter bufferWriter) + public static Utf8JsonWriter RentWriter(JsonSerializerOptions options, IBufferWriter bufferWriter) { ThreadLocalState state = t_threadLocalState ??= new(); Utf8JsonWriter writer; diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonTestHelper.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonTestHelper.cs index 3083d0bed99a0..53adc875c4e6a 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonTestHelper.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonTestHelper.cs @@ -132,6 +132,249 @@ public static byte[] SequenceReturnBytesHelper(byte[] data, out int length, Json return ReaderLoop(data.Length, out length, ref reader); } + public static byte[] SequenceReturnBytesHelper(byte[] data, out int length, int randomSeed, JsonCommentHandling commentHandling = JsonCommentHandling.Disallow, int maxDepth = 64) + { + ReadOnlySequence sequence = CreateSegmentsRandom(data, randomSeed); + var state = new JsonReaderState(new JsonReaderOptions { CommentHandling = commentHandling, MaxDepth = maxDepth }); + var reader = new Utf8JsonReader(sequence, true, state); + return ReaderLoop(data.Length, out length, ref reader); + } + + internal sealed class CustomMemoryForTest : IMemoryOwner + { + private bool _disposed; + private T[] _array; + private readonly int _offset; + private readonly int _length; + + public CustomMemoryForTest(T[] array) : this(array, 0, array.Length) + { + } + + public CustomMemoryForTest(T[] array, int offset, int length) + { + _array = array; + _offset = offset; + _length = length; + } + + public Memory Memory + { + get + { + if (_disposed) throw new ObjectDisposedException(this.GetType().Name); + return new Memory(_array, _offset, _length); + } + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _array = null!; + _disposed = true; + } + } + + internal sealed class BufferSegment : ReadOnlySequenceSegment + { + public BufferSegment(Memory memory) + { + Memory = memory; + } + + public BufferSegment Append(Memory memory) + { + var segment = new BufferSegment(memory) + { + RunningIndex = RunningIndex + Memory.Length + }; + Next = segment; + return segment; + } + } + + internal abstract class ReadOnlySequenceFactory + { + public static ReadOnlySequenceFactory ArrayFactory { get; } = new ArrayTestSequenceFactory(); + public static ReadOnlySequenceFactory MemoryFactory { get; } = new MemoryTestSequenceFactory(); + public static ReadOnlySequenceFactory OwnedMemoryFactory { get; } = new OwnedMemoryTestSequenceFactory(); + public static ReadOnlySequenceFactory SingleSegmentFactory { get; } = new SingleSegmentTestSequenceFactory(); + public static ReadOnlySequenceFactory SegmentPerByteFactory { get; } = new BytePerSegmentTestSequenceFactory(); + + public abstract ReadOnlySequence CreateOfSize(int size); + public abstract ReadOnlySequence CreateWithContent(byte[] data); + + public ReadOnlySequence CreateWithContent(string data) + { + return CreateWithContent(Encoding.ASCII.GetBytes(data)); + } + + internal sealed class ArrayTestSequenceFactory : ReadOnlySequenceFactory + { + public override ReadOnlySequence CreateOfSize(int size) + { + return new ReadOnlySequence(new byte[size + 20], 10, size); + } + + public override ReadOnlySequence CreateWithContent(byte[] data) + { + var startSegment = new byte[data.Length + 20]; + Array.Copy(data, 0, startSegment, 10, data.Length); + return new ReadOnlySequence(startSegment, 10, data.Length); + } + } + + internal sealed class MemoryTestSequenceFactory : ReadOnlySequenceFactory + { + public override ReadOnlySequence CreateOfSize(int size) + { + return CreateWithContent(new byte[size]); + } + + public override ReadOnlySequence CreateWithContent(byte[] data) + { + var startSegment = new byte[data.Length + 20]; + Array.Copy(data, 0, startSegment, 10, data.Length); + return new ReadOnlySequence(new Memory(startSegment, 10, data.Length)); + } + } + + internal sealed class OwnedMemoryTestSequenceFactory : ReadOnlySequenceFactory + { + public override ReadOnlySequence CreateOfSize(int size) + { + return CreateWithContent(new byte[size]); + } + + public override ReadOnlySequence CreateWithContent(byte[] data) + { + var startSegment = new byte[data.Length + 20]; + Array.Copy(data, 0, startSegment, 10, data.Length); + return new ReadOnlySequence(new CustomMemoryForTest(startSegment, 10, data.Length).Memory); + } + } + + internal sealed class SingleSegmentTestSequenceFactory : ReadOnlySequenceFactory + { + public override ReadOnlySequence CreateOfSize(int size) + { + return CreateWithContent(new byte[size]); + } + + public override ReadOnlySequence CreateWithContent(byte[] data) + { + return CreateSegments(data); + } + } + + internal sealed class BytePerSegmentTestSequenceFactory : ReadOnlySequenceFactory + { + public override ReadOnlySequence CreateOfSize(int size) + { + return CreateWithContent(new byte[size]); + } + + public override ReadOnlySequence CreateWithContent(byte[] data) + { + var segments = new List((data.Length * 2) + 1); + + segments.Add(Array.Empty()); + foreach (var b in data) + { + segments.Add(new[] { b }); + segments.Add(Array.Empty()); + } + + return CreateSegments(segments.ToArray()); + } + } + + internal sealed class RandomSegmentTestSequenceFactory : ReadOnlySequenceFactory + { + private int _rand; + public RandomSegmentTestSequenceFactory(int rand) + { + _rand = rand; + } + + public override ReadOnlySequence CreateOfSize(int size) + { + return CreateWithContent(new byte[size]); + } + + public override ReadOnlySequence CreateWithContent(byte[] data) + { + var segments = new List(); + + segments.Add(Array.Empty()); + var random = new Random(_rand); + var size = data.Length; + while (size > 0) + { + var segmentSize = random.Next(1, size); + var arr = new byte[segmentSize]; + data.AsSpan(data.Length - size, segmentSize).CopyTo(arr); + segments.Add(arr); + segments.Add(Array.Empty()); + size -= segmentSize; + } + //foreach (var b in data) + //{ + // segments.Add(new[] { b }); + // segments.Add(Array.Empty()); + //} + + return CreateSegments(segments.ToArray()); + } + } + + public static ReadOnlySequence CreateSegments(params byte[][] inputs) + { + if (inputs == null || inputs.Length == 0) + { + throw new InvalidOperationException(); + } + + int i = 0; + + BufferSegment? last = null; + BufferSegment? first = null; + + do + { + byte[] s = inputs[i]; + int length = s.Length; + int dataOffset = length; + var chars = new byte[length * 2]; + + for (int j = 0; j < length; j++) + { + chars[dataOffset + j] = s[j]; + } + + // Create a segment that has offset relative to the OwnedMemory and OwnedMemory itself has offset relative to array + var memory = new Memory(chars).Slice(length, length); + + if (first == null) + { + first = new BufferSegment(memory); + last = first; + } + else + { + last = last!.Append(memory); + } + i++; + } while (i < inputs.Length); + + return new ReadOnlySequence(first, 0, last, last.Memory.Length); + } + } + public delegate void Utf8JsonReaderAction(ref Utf8JsonReader reader); public static void AssertWithSingleAndMultiSegmentReader(string json, Utf8JsonReaderAction action, JsonReaderOptions options = default) @@ -157,6 +400,19 @@ public static ReadOnlySequence CreateSegments(byte[] data) return new ReadOnlySequence(firstSegment, 0, secondSegment, secondMem.Length); } + public static ReadOnlySequence CreateSegmentsRandom(byte[] data, int randomSeed) + { + return new ReadOnlySequenceFactory.RandomSegmentTestSequenceFactory(randomSeed).CreateWithContent(data); + //return ReadOnlySequenceFactory.SegmentPerByteFactory.CreateWithContent(data); + //ReadOnlyMemory dataMemory = data; + + //var firstSegment = new BufferSegment(dataMemory.Slice(0, data.Length / 2)); + //ReadOnlyMemory secondMem = dataMemory.Slice(data.Length / 2); + //BufferSegment secondSegment = firstSegment.Append(secondMem); + + //return new ReadOnlySequence(firstSegment, 0, secondSegment, secondMem.Length); + } + public static ReadOnlySequence CreateSegments(byte[] data, int splitLocation) { Debug.Assert(splitLocation <= data.Length); diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerApiValidation.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerApiValidation.cs index 1057e6a89006c..d3353e5364bfa 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerApiValidation.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerApiValidation.cs @@ -50,6 +50,11 @@ public class JsonSerializerApiValidation_Node : JsonSerializerApiValidation { public JsonSerializerApiValidation_Node() : base(JsonSerializerWrapper.NodeSerializer) { } } + + public class JsonSerializerApiValidation_Pipe : JsonSerializerApiValidation + { + public JsonSerializerApiValidation_Pipe() : base(JsonSerializerWrapper.PipeSerializer) { } + } } /// diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerWrapper.Reflection.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerWrapper.Reflection.cs index d3531970a8e76..9ec942863f615 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerWrapper.Reflection.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerWrapper.Reflection.cs @@ -1,10 +1,12 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.IO; +using System.IO.Pipelines; using System.Runtime.CompilerServices; using System.Text.Json.Nodes; using System.Text.Json.Serialization.Metadata; @@ -32,6 +34,7 @@ protected JsonSerializerWrapper() public static JsonSerializerWrapper DocumentSerializer { get; } = new DocumentSerializerWrapper(); public static JsonSerializerWrapper ElementSerializer { get; } = new ElementSerializerWrapper(); public static JsonSerializerWrapper NodeSerializer { get; } = new NodeSerializerWrapper(); + public static JsonSerializerWrapper PipeSerializer { get; } = new PipelinesSerializerWrapper(); private class SpanSerializerWrapper : JsonSerializerWrapper { @@ -881,5 +884,76 @@ private int ReadExactlyFromSource(byte[] buffer, int offset, int count) public override void SetLength(long value) => throw new NotSupportedException(); public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); } + + // TODO: Deserialize to use PipeReader overloads once implemented + private class PipelinesSerializerWrapper : JsonSerializerWrapper + { + public override JsonSerializerOptions DefaultOptions => JsonSerializerOptions.Default; + public override bool SupportsNullValueOnDeserialize => true; + + public override async Task DeserializeWrapper(string json, JsonSerializerOptions options = null) + { + return await JsonSerializer.DeserializeAsync(new MemoryStream(Encoding.UTF8.GetBytes(json)), options); + } + public override async Task DeserializeWrapper(string json, Type type, JsonSerializerOptions options = null) + { + return await JsonSerializer.DeserializeAsync(new MemoryStream(Encoding.UTF8.GetBytes(json)), type, options); + } + + public override async Task DeserializeWrapper(string json, JsonTypeInfo jsonTypeInfo) + { + return await JsonSerializer.DeserializeAsync(new MemoryStream(Encoding.UTF8.GetBytes(json)), jsonTypeInfo); + } + + public override async Task DeserializeWrapper(string value, JsonTypeInfo jsonTypeInfo) + { + return await JsonSerializer.DeserializeAsync(new MemoryStream(Encoding.UTF8.GetBytes(value)), jsonTypeInfo); + } + + public override async Task DeserializeWrapper(string json, Type type, JsonSerializerContext context) + { + return await JsonSerializer.DeserializeAsync(new MemoryStream(Encoding.UTF8.GetBytes(json)), type, context); + } + + public override async Task SerializeWrapper(object value, Type inputType, JsonSerializerOptions options = null) + { + Pipe pipe = new Pipe(); + await JsonSerializer.SerializeAsync(pipe.Writer, value, inputType, options); + ReadResult result = await pipe.Reader.ReadAsync(); + return Encoding.UTF8.GetString(result.Buffer.ToArray()); + } + + public override async Task SerializeWrapper(T value, JsonSerializerOptions options = null) + { + Pipe pipe = new Pipe(); + await JsonSerializer.SerializeAsync(pipe.Writer, value, options); + ReadResult result = await pipe.Reader.ReadAsync(); + return Encoding.UTF8.GetString(result.Buffer.ToArray()); + } + + public override async Task SerializeWrapper(object value, Type inputType, JsonSerializerContext context) + { + Pipe pipe = new Pipe(); + await JsonSerializer.SerializeAsync(pipe.Writer, value, inputType, context); + ReadResult result = await pipe.Reader.ReadAsync(); + return Encoding.UTF8.GetString(result.Buffer.ToArray()); + } + + public override async Task SerializeWrapper(T value, JsonTypeInfo jsonTypeInfo) + { + Pipe pipe = new Pipe(); + await JsonSerializer.SerializeAsync(pipe.Writer, value, jsonTypeInfo); + ReadResult result = await pipe.Reader.ReadAsync(); + return Encoding.UTF8.GetString(result.Buffer.ToArray()); + } + + public override async Task SerializeWrapper(object value, JsonTypeInfo jsonTypeInfo) + { + Pipe pipe = new Pipe(); + await JsonSerializer.SerializeAsync(pipe.Writer, value, jsonTypeInfo); + ReadResult result = await pipe.Reader.ReadAsync(); + return Encoding.UTF8.GetString(result.Buffer.ToArray()); + } + } } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/MetadataTests/MetadataTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/MetadataTests/MetadataTests.cs index 0e40f66a685dc..a9e943ff06856 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/MetadataTests/MetadataTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/MetadataTests/MetadataTests.cs @@ -47,6 +47,11 @@ public class MetadataTests_Node : MetadataTests public MetadataTests_Node() : base(JsonSerializerWrapper.NodeSerializer) { } } + public class MetadataTests_Pipe : MetadataTests + { + public MetadataTests_Pipe() : base(JsonSerializerWrapper.PipeSerializer) { } + } + public abstract partial class MetadataTests { protected JsonSerializerWrapper Serializer { get; } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/NumberHandlingTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/NumberHandlingTests.cs index 1dfa1e16b0b88..f1229f2c7e986 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/NumberHandlingTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/NumberHandlingTests.cs @@ -42,6 +42,11 @@ public class NumberHandlingTests_Node : NumberHandlingTests_OverloadSpecific public NumberHandlingTests_Node() : base(JsonSerializerWrapper.NodeSerializer) { } } + public class NumberHandlingTests_Pipe : NumberHandlingTests_OverloadSpecific + { + public NumberHandlingTests_Pipe() : base(JsonSerializerWrapper.PipeSerializer) { } + } + public abstract class NumberHandlingTests_OverloadSpecific { private JsonSerializerWrapper Serializer { get; } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/PolymorphicTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/PolymorphicTests.cs index 46f2202b1420e..9938a949e2dce 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/PolymorphicTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/PolymorphicTests.cs @@ -55,6 +55,11 @@ public class PolymorphicTests_Node : PolymorphicTests public PolymorphicTests_Node() : base(JsonSerializerWrapper.NodeSerializer) { } } + public class PolymorphicTests_Pipe : PolymorphicTests + { + public PolymorphicTests_Pipe() : base(JsonSerializerWrapper.PipeSerializer) { } + } + public abstract partial class PolymorphicTests : SerializerTests { public PolymorphicTests(JsonSerializerWrapper serializer) : base(serializer) diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/RequiredKeywordTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/RequiredKeywordTests.cs index 41dd4f61688f1..ce78182ee3262 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/RequiredKeywordTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/RequiredKeywordTests.cs @@ -52,4 +52,9 @@ public class RequiredKeywordTests_Node : RequiredKeywordTests { public RequiredKeywordTests_Node() : base(JsonSerializerWrapper.NodeSerializer) { } } + + public class RequiredKeywordTests_Pipe : RequiredKeywordTests + { + public RequiredKeywordTests_Pipe() : base(JsonSerializerWrapper.PipeSerializer) { } + } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonReaderTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonReaderTests.cs index 98fc08a82740c..aeca36b3e1e3c 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonReaderTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonReaderTests.cs @@ -225,54 +225,58 @@ public static void StateRecovery() [MemberData(nameof(TestCases))] public static void TestJsonReaderUtf8(bool compactData, TestCaseType type, string jsonString) { - // Remove all formatting/indendation - if (compactData) + for (var i = 0; i < 1; ++i) { - jsonString = JsonTestHelper.GetCompactString(jsonString); - } + var rand = new Random().Next(); + // Remove all formatting/indendation + if (compactData) + { + jsonString = JsonTestHelper.GetCompactString(jsonString); + } - byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString); + byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString); - SpanSequenceStatesAreEqual(dataUtf8); + SpanSequenceStatesAreEqual(dataUtf8); - byte[] result = JsonTestHelper.ReturnBytesHelper(dataUtf8, out int length); - string actualStr = Encoding.UTF8.GetString(result, 0, length); + byte[] result = JsonTestHelper.ReturnBytesHelper(dataUtf8, out int length); + string actualStr = Encoding.UTF8.GetString(result, 0, length); - byte[] resultSequence = JsonTestHelper.SequenceReturnBytesHelper(dataUtf8, out length); - string actualStrSequence = Encoding.UTF8.GetString(resultSequence, 0, length); + byte[] resultSequence = JsonTestHelper.SequenceReturnBytesHelper(dataUtf8, out length, rand); + string actualStrSequence = Encoding.UTF8.GetString(resultSequence, 0, length); - Stream stream = new MemoryStream(dataUtf8); - TextReader reader = new StreamReader(stream, Encoding.UTF8, false, 1024, true); - string expectedStr = JsonTestHelper.NewtonsoftReturnStringHelper(reader); + Stream stream = new MemoryStream(dataUtf8); + TextReader reader = new StreamReader(stream, Encoding.UTF8, false, 1024, true); + string expectedStr = JsonTestHelper.NewtonsoftReturnStringHelper(reader); - Assert.Equal(expectedStr, actualStr); - Assert.Equal(expectedStr, actualStrSequence); + Assert.Equal(expectedStr, actualStr); + Assert.Equal(expectedStr, actualStrSequence); - // Json payload contains numbers that are too large for .NET (need BigInteger+) - if (type != TestCaseType.FullSchema1 && type != TestCaseType.BasicLargeNum) - { - object jsonValues = JsonTestHelper.ReturnObjectHelper(dataUtf8); - string str = JsonTestHelper.ObjectToString(jsonValues); - ReadOnlySpan expectedSpan = expectedStr.AsSpan(0, expectedStr.Length - 2); - ReadOnlySpan actualSpan = str.AsSpan(0, str.Length - 2); - Assert.True(expectedSpan.SequenceEqual(actualSpan)); - } + // Json payload contains numbers that are too large for .NET (need BigInteger+) + if (type != TestCaseType.FullSchema1 && type != TestCaseType.BasicLargeNum) + { + object jsonValues = JsonTestHelper.ReturnObjectHelper(dataUtf8); + string str = JsonTestHelper.ObjectToString(jsonValues); + ReadOnlySpan expectedSpan = expectedStr.AsSpan(0, expectedStr.Length - 2); + ReadOnlySpan actualSpan = str.AsSpan(0, str.Length - 2); + Assert.True(expectedSpan.SequenceEqual(actualSpan)); + } - result = JsonTestHelper.ReturnBytesHelper(dataUtf8, out length, JsonCommentHandling.Skip); - actualStr = Encoding.UTF8.GetString(result, 0, length); - resultSequence = JsonTestHelper.SequenceReturnBytesHelper(dataUtf8, out length, JsonCommentHandling.Skip); - actualStrSequence = Encoding.UTF8.GetString(resultSequence, 0, length); + result = JsonTestHelper.ReturnBytesHelper(dataUtf8, out length, JsonCommentHandling.Skip); + actualStr = Encoding.UTF8.GetString(result, 0, length); + resultSequence = JsonTestHelper.SequenceReturnBytesHelper(dataUtf8, out length, rand, JsonCommentHandling.Skip); + actualStrSequence = Encoding.UTF8.GetString(resultSequence, 0, length); - Assert.Equal(expectedStr, actualStr); - Assert.Equal(expectedStr, actualStrSequence); + Assert.Equal(expectedStr, actualStr); + Assert.Equal(expectedStr, actualStrSequence); - result = JsonTestHelper.ReturnBytesHelper(dataUtf8, out length, JsonCommentHandling.Allow); - actualStr = Encoding.UTF8.GetString(result, 0, length); - resultSequence = JsonTestHelper.SequenceReturnBytesHelper(dataUtf8, out length, JsonCommentHandling.Allow); - actualStrSequence = Encoding.UTF8.GetString(resultSequence, 0, length); + result = JsonTestHelper.ReturnBytesHelper(dataUtf8, out length, JsonCommentHandling.Allow); + actualStr = Encoding.UTF8.GetString(result, 0, length); + resultSequence = JsonTestHelper.SequenceReturnBytesHelper(dataUtf8, out length, rand, JsonCommentHandling.Allow); + actualStrSequence = Encoding.UTF8.GetString(resultSequence, 0, length); - Assert.Equal(expectedStr, actualStr); - Assert.Equal(expectedStr, actualStrSequence); + Assert.Equal(expectedStr, actualStr); + Assert.Equal(expectedStr, actualStrSequence); + } } [Theory] From 4efef1f39934f42a05c1258269afc033ed903a6b Mon Sep 17 00:00:00 2001 From: Brennan Date: Mon, 22 Apr 2024 17:38:50 -0700 Subject: [PATCH 03/14] organize and tests --- .../src/System.Text.Json.csproj | 1 + .../JsonSerializer.Write.Pipe.cs | 194 ++++++++++++++++++ .../JsonSerializer.Write.Stream.cs | 143 ------------- .../Metadata/JsonTypeInfoOfT.WriteHelpers.cs | 3 +- .../Serialization/CollectionTests.cs | 5 + .../Serialization/ConstructorTests.cs | 6 + .../Serialization/InvalidTypeTests.cs | 5 + .../JsonCreationHandlingTests.cs | 5 + .../JsonSerializerWrapper.Reflection.cs | 2 +- .../Serialization/ReferenceHandlerTests.cs | 5 + .../UnmappedMemberHandlingTests.cs | 5 + 11 files changed, 228 insertions(+), 146 deletions(-) create mode 100644 src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Pipe.cs diff --git a/src/libraries/System.Text.Json/src/System.Text.Json.csproj b/src/libraries/System.Text.Json/src/System.Text.Json.csproj index 57f25a2bfb0c2..15d23ec13533d 100644 --- a/src/libraries/System.Text.Json/src/System.Text.Json.csproj +++ b/src/libraries/System.Text.Json/src/System.Text.Json.csproj @@ -252,6 +252,7 @@ The System.Text.Json library is built-in as part of the shared framework in .NET + diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Pipe.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Pipe.cs new file mode 100644 index 0000000000000..dbd8debe55f09 --- /dev/null +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Pipe.cs @@ -0,0 +1,194 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using System.IO.Pipelines; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; +using System.Threading; +using System.Threading.Tasks; + +namespace System.Text.Json +{ + public static partial class JsonSerializer + { + /// + /// Converts the provided value to UTF-8 encoded JSON text and write it to the . + /// + /// The type of the value to serialize. + /// The UTF-8 to write to. + /// The value to convert. + /// Metadata about the type to convert. + /// The that can be used to cancel the write operation. + /// A task that represents the asynchronous write operation. + /// + /// is . + /// + public static Task SerializeAsync( + PipeWriter utf8Json, + TValue value, + JsonTypeInfo jsonTypeInfo, + CancellationToken cancellationToken = default) + { + if (utf8Json is null) + { + ThrowHelper.ThrowArgumentNullException(nameof(utf8Json)); + } + if (jsonTypeInfo is null) + { + ThrowHelper.ThrowArgumentNullException(nameof(jsonTypeInfo)); + } + + jsonTypeInfo.EnsureConfigured(); + return jsonTypeInfo.SerializeAsync(utf8Json, value, cancellationToken); + } + + /// + /// Converts the provided value to UTF-8 encoded JSON text and write it to the . + /// + /// The type of the value to serialize. + /// The UTF-8 to write to. + /// The value to convert. + /// Options to control the conversion behavior. + /// The that can be used to cancel the write operation. + /// A task that represents the asynchronous write operation. + /// + /// is . + /// + /// + /// There is no compatible + /// for or its serializable members. + /// + [RequiresUnreferencedCode(SerializationUnreferencedCodeMessage)] + [RequiresDynamicCode(SerializationRequiresDynamicCodeMessage)] + public static Task SerializeAsync( + PipeWriter utf8Json, + TValue value, + JsonSerializerOptions? options = null, + CancellationToken cancellationToken = default) + { + if (utf8Json is null) + { + ThrowHelper.ThrowArgumentNullException(nameof(utf8Json)); + } + + JsonTypeInfo jsonTypeInfo = GetTypeInfo(options); + return jsonTypeInfo.SerializeAsync(utf8Json, value, cancellationToken); + } + + /// + /// Converts the provided value to UTF-8 encoded JSON text and write it to the . + /// + /// The UTF-8 to write to. + /// The value to convert. + /// Metadata about the type to convert. + /// The that can be used to cancel the write operation. + /// A task that represents the asynchronous write operation. + /// + /// is . + /// + /// + /// does not match the type of . + /// + public static Task SerializeAsync( + PipeWriter utf8Json, + object? value, + JsonTypeInfo jsonTypeInfo, + CancellationToken cancellationToken = default) + { + if (utf8Json is null) + { + ThrowHelper.ThrowArgumentNullException(nameof(utf8Json)); + } + + if (jsonTypeInfo is null) + { + ThrowHelper.ThrowArgumentNullException(nameof(jsonTypeInfo)); + } + + jsonTypeInfo.EnsureConfigured(); + return jsonTypeInfo.SerializeAsObjectAsync(utf8Json, value, cancellationToken); + } + + /// + /// Converts the provided value to UTF-8 encoded JSON text and write it to the . + /// + /// The UTF-8 to write to. + /// The value to convert. + /// The type of the to convert. + /// A metadata provider for serializable types. + /// The that can be used to cancel the write operation. + /// A task that represents the asynchronous write operation. + /// + /// is not compatible with . + /// + /// + /// , , or is . + /// + /// + /// There is no compatible + /// for or its serializable members. + /// + public static Task SerializeAsync( + PipeWriter utf8Json, + object? value, + Type inputType, + JsonSerializerContext context, + CancellationToken cancellationToken = default) + { + if (utf8Json is null) + { + ThrowHelper.ThrowArgumentNullException(nameof(utf8Json)); + } + + if (context is null) + { + ThrowHelper.ThrowArgumentNullException(nameof(context)); + } + + ValidateInputType(value, inputType); + JsonTypeInfo jsonTypeInfo = GetTypeInfo(context, inputType); + + return jsonTypeInfo.SerializeAsObjectAsync(utf8Json, value, cancellationToken); + } + + /// + /// Converts the provided value to UTF-8 encoded JSON text and write it to the . + /// + /// The UTF-8 to write to. + /// The value to convert. + /// The type of the to convert. + /// Options to control the conversion behavior. + /// The that can be used to cancel the write operation. + /// A task that represents the asynchronous write operation. + /// + /// is not compatible with . + /// + /// + /// or is . + /// + /// + /// There is no compatible + /// for or its serializable members. + /// + [RequiresUnreferencedCode(SerializationUnreferencedCodeMessage)] + [RequiresDynamicCode(SerializationRequiresDynamicCodeMessage)] + public static Task SerializeAsync( + PipeWriter utf8Json, + object? value, + Type inputType, + JsonSerializerOptions? options = null, + CancellationToken cancellationToken = default) + { + if (utf8Json is null) + { + ThrowHelper.ThrowArgumentNullException(nameof(utf8Json)); + } + + ValidateInputType(value, inputType); + JsonTypeInfo jsonTypeInfo = GetTypeInfo(options, inputType); + + return jsonTypeInfo.SerializeAsObjectAsync(utf8Json, value, cancellationToken); + } + } +} diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs index 0a270fb9be8fc..7f9412280ec96 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs @@ -54,121 +54,6 @@ public static Task SerializeAsync( return jsonTypeInfo.SerializeAsync(utf8Json, value, cancellationToken); } - /// - /// g - /// - /// - /// - /// - /// - /// - /// - [RequiresUnreferencedCode(SerializationUnreferencedCodeMessage)] - [RequiresDynamicCode(SerializationRequiresDynamicCodeMessage)] - public static Task SerializeAsync( - PipeWriter utf8Json, - TValue value, - JsonSerializerOptions? options = null, - CancellationToken cancellationToken = default) - { - if (utf8Json is null) - { - ThrowHelper.ThrowArgumentNullException(nameof(utf8Json)); - } - - JsonTypeInfo jsonTypeInfo = GetTypeInfo(options); - return jsonTypeInfo.SerializeAsync(utf8Json, value, cancellationToken); - } - - /// - /// f - /// - /// - /// - /// - /// - /// - public static Task SerializeAsync( - PipeWriter utf8Json, - object? value, - JsonTypeInfo jsonTypeInfo, - CancellationToken cancellationToken = default) - { - if (utf8Json is null) - { - ThrowHelper.ThrowArgumentNullException(nameof(utf8Json)); - } - - if (jsonTypeInfo is null) - { - ThrowHelper.ThrowArgumentNullException(nameof(jsonTypeInfo)); - } - - jsonTypeInfo.EnsureConfigured(); - return jsonTypeInfo.SerializeAsObjectAsync(utf8Json, value, cancellationToken); - } - - /// - /// g - /// - /// - /// - /// - /// - /// - /// - public static Task SerializeAsync( - PipeWriter utf8Json, - object? value, - Type inputType, - JsonSerializerContext context, - CancellationToken cancellationToken = default) - { - if (utf8Json is null) - { - ThrowHelper.ThrowArgumentNullException(nameof(utf8Json)); - } - - if (context is null) - { - ThrowHelper.ThrowArgumentNullException(nameof(context)); - } - - ValidateInputType(value, inputType); - JsonTypeInfo jsonTypeInfo = GetTypeInfo(context, inputType); - - return jsonTypeInfo.SerializeAsObjectAsync(utf8Json, value, cancellationToken); - } - - /// - /// g - /// - /// - /// - /// - /// - /// - /// - [RequiresUnreferencedCode(SerializationUnreferencedCodeMessage)] - [RequiresDynamicCode(SerializationRequiresDynamicCodeMessage)] - public static Task SerializeAsync( - PipeWriter utf8Json, - object? value, - Type inputType, - JsonSerializerOptions? options = null, - CancellationToken cancellationToken = default) - { - if (utf8Json is null) - { - ThrowHelper.ThrowArgumentNullException(nameof(utf8Json)); - } - - ValidateInputType(value, inputType); - JsonTypeInfo jsonTypeInfo = GetTypeInfo(options, inputType); - - return jsonTypeInfo.SerializeAsObjectAsync(utf8Json, value, cancellationToken); - } - /// /// Converts the provided value to UTF-8 encoded JSON text and write it to the . /// @@ -303,34 +188,6 @@ public static Task SerializeAsync( return jsonTypeInfo.SerializeAsync(utf8Json, value, cancellationToken); } - /// - /// tmp - /// - /// - /// - /// - /// - /// - /// - public static Task SerializeAsync( - PipeWriter utf8Json, - TValue value, - JsonTypeInfo jsonTypeInfo, - CancellationToken cancellationToken = default) - { - if (utf8Json is null) - { - ThrowHelper.ThrowArgumentNullException(nameof(utf8Json)); - } - if (jsonTypeInfo is null) - { - ThrowHelper.ThrowArgumentNullException(nameof(jsonTypeInfo)); - } - - jsonTypeInfo.EnsureConfigured(); - return jsonTypeInfo.SerializeAsync(utf8Json, value, cancellationToken); - } - /// /// Converts the provided value to UTF-8 encoded JSON text and write it to the . /// diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs index f9ea92bd8fa25..8addd2b307bb5 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs @@ -245,14 +245,13 @@ rootValue is not null && state.CancellationToken = cancellationToken; - //using var bufferWriter = new PooledByteBufferWriter(Options.DefaultBufferSize); using var writer = new Utf8JsonWriter(utf8Json, Options.GetWriterOptions()); try { do { - state.FlushThreshold = 4096;//(int)(bufferWriter.Capacity * JsonSerializer.FlushThreshold); + state.FlushThreshold = (int)(4096 * JsonSerializer.FlushThreshold); try { diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/CollectionTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/CollectionTests.cs index 3b3d3ac1942d5..6589f6cc9edbb 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/CollectionTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/CollectionTests.cs @@ -24,4 +24,9 @@ public sealed partial class CollectionTestsDynamic_SyncStream : CollectionTests { public CollectionTestsDynamic_SyncStream() : base(JsonSerializerWrapper.SyncStreamSerializer) { } } + + public sealed partial class CollectionTestsDynamic_Pipe : CollectionTests + { + public CollectionTestsDynamic_Pipe() : base(JsonSerializerWrapper.PipeSerializer) { } + } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ConstructorTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ConstructorTests.cs index 7808e5b109b48..202bcab22599a 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ConstructorTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ConstructorTests.cs @@ -27,4 +27,10 @@ public class ConstructorTests_Span : ConstructorTests public ConstructorTests_Span() : base(JsonSerializerWrapper.SpanSerializer) { } } + + public class ConstructorTests_Pipe : ConstructorTests + { + public ConstructorTests_Pipe() + : base(JsonSerializerWrapper.PipeSerializer) { } + } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/InvalidTypeTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/InvalidTypeTests.cs index 91ca5c015a64d..e09e4a00c8e2a 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/InvalidTypeTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/InvalidTypeTests.cs @@ -37,6 +37,11 @@ public class InvalidTypeTests_Writer : InvalidTypeTests public InvalidTypeTests_Writer() : base(JsonSerializerWrapper.ReaderWriterSerializer) { } } + public class InvalidTypeTests_Pipe : InvalidTypeTests + { + public InvalidTypeTests_Pipe() : base(JsonSerializerWrapper.PipeSerializer) { } + } + public abstract class InvalidTypeTests { private JsonSerializerWrapper Serializer { get; } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonCreationHandlingTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonCreationHandlingTests.cs index 5a669bb5b2f90..7cd83ab63944a 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonCreationHandlingTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonCreationHandlingTests.cs @@ -22,4 +22,9 @@ public sealed class JsonCreationHandlingTests_SyncStream : JsonCreationHandlingT { public JsonCreationHandlingTests_SyncStream() : base(JsonSerializerWrapper.SyncStreamSerializer) { } } + + public sealed class JsonCreationHandlingTests_Pipe : JsonCreationHandlingTests + { + public JsonCreationHandlingTests_Pipe() : base(JsonSerializerWrapper.PipeSerializer) { } + } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerWrapper.Reflection.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerWrapper.Reflection.cs index 9ec942863f615..38230ae992cf2 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerWrapper.Reflection.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerWrapper.Reflection.cs @@ -926,7 +926,7 @@ public override async Task SerializeWrapper(object value, Type inputType public override async Task SerializeWrapper(T value, JsonSerializerOptions options = null) { Pipe pipe = new Pipe(); - await JsonSerializer.SerializeAsync(pipe.Writer, value, options); + await JsonSerializer.SerializeAsync(pipe.Writer, value, options); ReadResult result = await pipe.Reader.ReadAsync(); return Encoding.UTF8.GetString(result.Buffer.ToArray()); } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ReferenceHandlerTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ReferenceHandlerTests.cs index 3cfcdbeff8df5..21781b06f6660 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ReferenceHandlerTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ReferenceHandlerTests.cs @@ -29,4 +29,9 @@ public sealed class ReferenceHandlerTestsDynamic_IgnoreCycles_AsyncStream : Refe { public ReferenceHandlerTestsDynamic_IgnoreCycles_AsyncStream() : base(JsonSerializerWrapper.AsyncStreamSerializer) { } } + + public sealed class ReferenceHandlerTestsDynamic_IgnoreCycles_Pipe : ReferenceHandlerTests_IgnoreCycles + { + public ReferenceHandlerTestsDynamic_IgnoreCycles_Pipe() : base(JsonSerializerWrapper.PipeSerializer) { } + } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/UnmappedMemberHandlingTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/UnmappedMemberHandlingTests.cs index 2b532b6e6f44b..8434b3b8444f2 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/UnmappedMemberHandlingTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/UnmappedMemberHandlingTests.cs @@ -12,4 +12,9 @@ public sealed partial class UnmappedMemberHandlingTests_AsyncStream : UnmappedMe { public UnmappedMemberHandlingTests_AsyncStream() : base(JsonSerializerWrapper.AsyncStreamSerializer) { } } + + public sealed partial class UnmappedMemberHandlingTests_Pipe : UnmappedMemberHandlingTests + { + public UnmappedMemberHandlingTests_Pipe() : base(JsonSerializerWrapper.PipeSerializer) { } + } } From 5772e3fe9d71301d603b85348cfe55a01188921a Mon Sep 17 00:00:00 2001 From: Brennan Date: Mon, 22 Apr 2024 20:34:55 -0700 Subject: [PATCH 04/14] cleanup --- .../JsonSerializer.Write.Stream.cs | 1 - .../System.Text.Json.Tests/JsonTestHelper.cs | 256 ------------------ .../Utf8JsonReaderTests.cs | 76 +++--- 3 files changed, 36 insertions(+), 297 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs index 7f9412280ec96..9c678e37ad394 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Write.Stream.cs @@ -3,7 +3,6 @@ using System.Diagnostics.CodeAnalysis; using System.IO; -using System.IO.Pipelines; using System.Text.Json.Serialization; using System.Text.Json.Serialization.Metadata; using System.Threading; diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonTestHelper.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonTestHelper.cs index 53adc875c4e6a..3083d0bed99a0 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonTestHelper.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonTestHelper.cs @@ -132,249 +132,6 @@ public static byte[] SequenceReturnBytesHelper(byte[] data, out int length, Json return ReaderLoop(data.Length, out length, ref reader); } - public static byte[] SequenceReturnBytesHelper(byte[] data, out int length, int randomSeed, JsonCommentHandling commentHandling = JsonCommentHandling.Disallow, int maxDepth = 64) - { - ReadOnlySequence sequence = CreateSegmentsRandom(data, randomSeed); - var state = new JsonReaderState(new JsonReaderOptions { CommentHandling = commentHandling, MaxDepth = maxDepth }); - var reader = new Utf8JsonReader(sequence, true, state); - return ReaderLoop(data.Length, out length, ref reader); - } - - internal sealed class CustomMemoryForTest : IMemoryOwner - { - private bool _disposed; - private T[] _array; - private readonly int _offset; - private readonly int _length; - - public CustomMemoryForTest(T[] array) : this(array, 0, array.Length) - { - } - - public CustomMemoryForTest(T[] array, int offset, int length) - { - _array = array; - _offset = offset; - _length = length; - } - - public Memory Memory - { - get - { - if (_disposed) throw new ObjectDisposedException(this.GetType().Name); - return new Memory(_array, _offset, _length); - } - } - - public void Dispose() - { - if (_disposed) - { - return; - } - - _array = null!; - _disposed = true; - } - } - - internal sealed class BufferSegment : ReadOnlySequenceSegment - { - public BufferSegment(Memory memory) - { - Memory = memory; - } - - public BufferSegment Append(Memory memory) - { - var segment = new BufferSegment(memory) - { - RunningIndex = RunningIndex + Memory.Length - }; - Next = segment; - return segment; - } - } - - internal abstract class ReadOnlySequenceFactory - { - public static ReadOnlySequenceFactory ArrayFactory { get; } = new ArrayTestSequenceFactory(); - public static ReadOnlySequenceFactory MemoryFactory { get; } = new MemoryTestSequenceFactory(); - public static ReadOnlySequenceFactory OwnedMemoryFactory { get; } = new OwnedMemoryTestSequenceFactory(); - public static ReadOnlySequenceFactory SingleSegmentFactory { get; } = new SingleSegmentTestSequenceFactory(); - public static ReadOnlySequenceFactory SegmentPerByteFactory { get; } = new BytePerSegmentTestSequenceFactory(); - - public abstract ReadOnlySequence CreateOfSize(int size); - public abstract ReadOnlySequence CreateWithContent(byte[] data); - - public ReadOnlySequence CreateWithContent(string data) - { - return CreateWithContent(Encoding.ASCII.GetBytes(data)); - } - - internal sealed class ArrayTestSequenceFactory : ReadOnlySequenceFactory - { - public override ReadOnlySequence CreateOfSize(int size) - { - return new ReadOnlySequence(new byte[size + 20], 10, size); - } - - public override ReadOnlySequence CreateWithContent(byte[] data) - { - var startSegment = new byte[data.Length + 20]; - Array.Copy(data, 0, startSegment, 10, data.Length); - return new ReadOnlySequence(startSegment, 10, data.Length); - } - } - - internal sealed class MemoryTestSequenceFactory : ReadOnlySequenceFactory - { - public override ReadOnlySequence CreateOfSize(int size) - { - return CreateWithContent(new byte[size]); - } - - public override ReadOnlySequence CreateWithContent(byte[] data) - { - var startSegment = new byte[data.Length + 20]; - Array.Copy(data, 0, startSegment, 10, data.Length); - return new ReadOnlySequence(new Memory(startSegment, 10, data.Length)); - } - } - - internal sealed class OwnedMemoryTestSequenceFactory : ReadOnlySequenceFactory - { - public override ReadOnlySequence CreateOfSize(int size) - { - return CreateWithContent(new byte[size]); - } - - public override ReadOnlySequence CreateWithContent(byte[] data) - { - var startSegment = new byte[data.Length + 20]; - Array.Copy(data, 0, startSegment, 10, data.Length); - return new ReadOnlySequence(new CustomMemoryForTest(startSegment, 10, data.Length).Memory); - } - } - - internal sealed class SingleSegmentTestSequenceFactory : ReadOnlySequenceFactory - { - public override ReadOnlySequence CreateOfSize(int size) - { - return CreateWithContent(new byte[size]); - } - - public override ReadOnlySequence CreateWithContent(byte[] data) - { - return CreateSegments(data); - } - } - - internal sealed class BytePerSegmentTestSequenceFactory : ReadOnlySequenceFactory - { - public override ReadOnlySequence CreateOfSize(int size) - { - return CreateWithContent(new byte[size]); - } - - public override ReadOnlySequence CreateWithContent(byte[] data) - { - var segments = new List((data.Length * 2) + 1); - - segments.Add(Array.Empty()); - foreach (var b in data) - { - segments.Add(new[] { b }); - segments.Add(Array.Empty()); - } - - return CreateSegments(segments.ToArray()); - } - } - - internal sealed class RandomSegmentTestSequenceFactory : ReadOnlySequenceFactory - { - private int _rand; - public RandomSegmentTestSequenceFactory(int rand) - { - _rand = rand; - } - - public override ReadOnlySequence CreateOfSize(int size) - { - return CreateWithContent(new byte[size]); - } - - public override ReadOnlySequence CreateWithContent(byte[] data) - { - var segments = new List(); - - segments.Add(Array.Empty()); - var random = new Random(_rand); - var size = data.Length; - while (size > 0) - { - var segmentSize = random.Next(1, size); - var arr = new byte[segmentSize]; - data.AsSpan(data.Length - size, segmentSize).CopyTo(arr); - segments.Add(arr); - segments.Add(Array.Empty()); - size -= segmentSize; - } - //foreach (var b in data) - //{ - // segments.Add(new[] { b }); - // segments.Add(Array.Empty()); - //} - - return CreateSegments(segments.ToArray()); - } - } - - public static ReadOnlySequence CreateSegments(params byte[][] inputs) - { - if (inputs == null || inputs.Length == 0) - { - throw new InvalidOperationException(); - } - - int i = 0; - - BufferSegment? last = null; - BufferSegment? first = null; - - do - { - byte[] s = inputs[i]; - int length = s.Length; - int dataOffset = length; - var chars = new byte[length * 2]; - - for (int j = 0; j < length; j++) - { - chars[dataOffset + j] = s[j]; - } - - // Create a segment that has offset relative to the OwnedMemory and OwnedMemory itself has offset relative to array - var memory = new Memory(chars).Slice(length, length); - - if (first == null) - { - first = new BufferSegment(memory); - last = first; - } - else - { - last = last!.Append(memory); - } - i++; - } while (i < inputs.Length); - - return new ReadOnlySequence(first, 0, last, last.Memory.Length); - } - } - public delegate void Utf8JsonReaderAction(ref Utf8JsonReader reader); public static void AssertWithSingleAndMultiSegmentReader(string json, Utf8JsonReaderAction action, JsonReaderOptions options = default) @@ -400,19 +157,6 @@ public static ReadOnlySequence CreateSegments(byte[] data) return new ReadOnlySequence(firstSegment, 0, secondSegment, secondMem.Length); } - public static ReadOnlySequence CreateSegmentsRandom(byte[] data, int randomSeed) - { - return new ReadOnlySequenceFactory.RandomSegmentTestSequenceFactory(randomSeed).CreateWithContent(data); - //return ReadOnlySequenceFactory.SegmentPerByteFactory.CreateWithContent(data); - //ReadOnlyMemory dataMemory = data; - - //var firstSegment = new BufferSegment(dataMemory.Slice(0, data.Length / 2)); - //ReadOnlyMemory secondMem = dataMemory.Slice(data.Length / 2); - //BufferSegment secondSegment = firstSegment.Append(secondMem); - - //return new ReadOnlySequence(firstSegment, 0, secondSegment, secondMem.Length); - } - public static ReadOnlySequence CreateSegments(byte[] data, int splitLocation) { Debug.Assert(splitLocation <= data.Length); diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonReaderTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonReaderTests.cs index aeca36b3e1e3c..98fc08a82740c 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonReaderTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonReaderTests.cs @@ -225,58 +225,54 @@ public static void StateRecovery() [MemberData(nameof(TestCases))] public static void TestJsonReaderUtf8(bool compactData, TestCaseType type, string jsonString) { - for (var i = 0; i < 1; ++i) + // Remove all formatting/indendation + if (compactData) { - var rand = new Random().Next(); - // Remove all formatting/indendation - if (compactData) - { - jsonString = JsonTestHelper.GetCompactString(jsonString); - } + jsonString = JsonTestHelper.GetCompactString(jsonString); + } - byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString); + byte[] dataUtf8 = Encoding.UTF8.GetBytes(jsonString); - SpanSequenceStatesAreEqual(dataUtf8); + SpanSequenceStatesAreEqual(dataUtf8); - byte[] result = JsonTestHelper.ReturnBytesHelper(dataUtf8, out int length); - string actualStr = Encoding.UTF8.GetString(result, 0, length); + byte[] result = JsonTestHelper.ReturnBytesHelper(dataUtf8, out int length); + string actualStr = Encoding.UTF8.GetString(result, 0, length); - byte[] resultSequence = JsonTestHelper.SequenceReturnBytesHelper(dataUtf8, out length, rand); - string actualStrSequence = Encoding.UTF8.GetString(resultSequence, 0, length); + byte[] resultSequence = JsonTestHelper.SequenceReturnBytesHelper(dataUtf8, out length); + string actualStrSequence = Encoding.UTF8.GetString(resultSequence, 0, length); - Stream stream = new MemoryStream(dataUtf8); - TextReader reader = new StreamReader(stream, Encoding.UTF8, false, 1024, true); - string expectedStr = JsonTestHelper.NewtonsoftReturnStringHelper(reader); + Stream stream = new MemoryStream(dataUtf8); + TextReader reader = new StreamReader(stream, Encoding.UTF8, false, 1024, true); + string expectedStr = JsonTestHelper.NewtonsoftReturnStringHelper(reader); - Assert.Equal(expectedStr, actualStr); - Assert.Equal(expectedStr, actualStrSequence); + Assert.Equal(expectedStr, actualStr); + Assert.Equal(expectedStr, actualStrSequence); - // Json payload contains numbers that are too large for .NET (need BigInteger+) - if (type != TestCaseType.FullSchema1 && type != TestCaseType.BasicLargeNum) - { - object jsonValues = JsonTestHelper.ReturnObjectHelper(dataUtf8); - string str = JsonTestHelper.ObjectToString(jsonValues); - ReadOnlySpan expectedSpan = expectedStr.AsSpan(0, expectedStr.Length - 2); - ReadOnlySpan actualSpan = str.AsSpan(0, str.Length - 2); - Assert.True(expectedSpan.SequenceEqual(actualSpan)); - } + // Json payload contains numbers that are too large for .NET (need BigInteger+) + if (type != TestCaseType.FullSchema1 && type != TestCaseType.BasicLargeNum) + { + object jsonValues = JsonTestHelper.ReturnObjectHelper(dataUtf8); + string str = JsonTestHelper.ObjectToString(jsonValues); + ReadOnlySpan expectedSpan = expectedStr.AsSpan(0, expectedStr.Length - 2); + ReadOnlySpan actualSpan = str.AsSpan(0, str.Length - 2); + Assert.True(expectedSpan.SequenceEqual(actualSpan)); + } - result = JsonTestHelper.ReturnBytesHelper(dataUtf8, out length, JsonCommentHandling.Skip); - actualStr = Encoding.UTF8.GetString(result, 0, length); - resultSequence = JsonTestHelper.SequenceReturnBytesHelper(dataUtf8, out length, rand, JsonCommentHandling.Skip); - actualStrSequence = Encoding.UTF8.GetString(resultSequence, 0, length); + result = JsonTestHelper.ReturnBytesHelper(dataUtf8, out length, JsonCommentHandling.Skip); + actualStr = Encoding.UTF8.GetString(result, 0, length); + resultSequence = JsonTestHelper.SequenceReturnBytesHelper(dataUtf8, out length, JsonCommentHandling.Skip); + actualStrSequence = Encoding.UTF8.GetString(resultSequence, 0, length); - Assert.Equal(expectedStr, actualStr); - Assert.Equal(expectedStr, actualStrSequence); + Assert.Equal(expectedStr, actualStr); + Assert.Equal(expectedStr, actualStrSequence); - result = JsonTestHelper.ReturnBytesHelper(dataUtf8, out length, JsonCommentHandling.Allow); - actualStr = Encoding.UTF8.GetString(result, 0, length); - resultSequence = JsonTestHelper.SequenceReturnBytesHelper(dataUtf8, out length, rand, JsonCommentHandling.Allow); - actualStrSequence = Encoding.UTF8.GetString(resultSequence, 0, length); + result = JsonTestHelper.ReturnBytesHelper(dataUtf8, out length, JsonCommentHandling.Allow); + actualStr = Encoding.UTF8.GetString(result, 0, length); + resultSequence = JsonTestHelper.SequenceReturnBytesHelper(dataUtf8, out length, JsonCommentHandling.Allow); + actualStrSequence = Encoding.UTF8.GetString(resultSequence, 0, length); - Assert.Equal(expectedStr, actualStr); - Assert.Equal(expectedStr, actualStrSequence); - } + Assert.Equal(expectedStr, actualStr); + Assert.Equal(expectedStr, actualStrSequence); } [Theory] From 94c4be4862b141f6214c24b34f15876ecad7d9cf Mon Sep 17 00:00:00 2001 From: Brennan Date: Tue, 23 Apr 2024 16:57:47 -0700 Subject: [PATCH 05/14] FlushThreshold --- .../Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs index 8addd2b307bb5..2efb3b4617c62 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs @@ -251,7 +251,8 @@ rootValue is not null && { do { - state.FlushThreshold = (int)(4096 * JsonSerializer.FlushThreshold); + // Arbitrary threshold, default Pipe buffers are 4k so this allows 4 buffers to be created and filled (mostly) before flushing. + state.FlushThreshold = (int)(16384 * JsonSerializer.FlushThreshold); try { From 04f53d38f9f3ad9d1e74e506614b933f08c74d21 Mon Sep 17 00:00:00 2001 From: Brennan Date: Tue, 23 Apr 2024 17:07:47 -0700 Subject: [PATCH 06/14] cleanup --- .../Metadata/JsonTypeInfoOfT.WriteHelpers.cs | 1 - .../JsonSerializerWrapper.Reflection.cs | 25 +++++++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs index 2efb3b4617c62..6a31f3bd1983c 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs @@ -315,7 +315,6 @@ rootValue is not null && } } - // Root serialization method for non-async streaming serialization internal void Serialize( Stream utf8Json, diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerWrapper.Reflection.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerWrapper.Reflection.cs index 38230ae992cf2..6282c24e38249 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerWrapper.Reflection.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerWrapper.Reflection.cs @@ -920,7 +920,10 @@ public override async Task SerializeWrapper(object value, Type inputType Pipe pipe = new Pipe(); await JsonSerializer.SerializeAsync(pipe.Writer, value, inputType, options); ReadResult result = await pipe.Reader.ReadAsync(); - return Encoding.UTF8.GetString(result.Buffer.ToArray()); + + string stringResult = Encoding.UTF8.GetString(result.Buffer.ToArray()); + pipe.Reader.AdvanceTo(result.Buffer.End); + return stringResult; } public override async Task SerializeWrapper(T value, JsonSerializerOptions options = null) @@ -928,7 +931,10 @@ public override async Task SerializeWrapper(T value, JsonSerializerOp Pipe pipe = new Pipe(); await JsonSerializer.SerializeAsync(pipe.Writer, value, options); ReadResult result = await pipe.Reader.ReadAsync(); - return Encoding.UTF8.GetString(result.Buffer.ToArray()); + + string stringResult = Encoding.UTF8.GetString(result.Buffer.ToArray()); + pipe.Reader.AdvanceTo(result.Buffer.End); + return stringResult; } public override async Task SerializeWrapper(object value, Type inputType, JsonSerializerContext context) @@ -936,7 +942,10 @@ public override async Task SerializeWrapper(object value, Type inputType Pipe pipe = new Pipe(); await JsonSerializer.SerializeAsync(pipe.Writer, value, inputType, context); ReadResult result = await pipe.Reader.ReadAsync(); - return Encoding.UTF8.GetString(result.Buffer.ToArray()); + + string stringResult = Encoding.UTF8.GetString(result.Buffer.ToArray()); + pipe.Reader.AdvanceTo(result.Buffer.End); + return stringResult; } public override async Task SerializeWrapper(T value, JsonTypeInfo jsonTypeInfo) @@ -944,7 +953,10 @@ public override async Task SerializeWrapper(T value, JsonTypeInfo Pipe pipe = new Pipe(); await JsonSerializer.SerializeAsync(pipe.Writer, value, jsonTypeInfo); ReadResult result = await pipe.Reader.ReadAsync(); - return Encoding.UTF8.GetString(result.Buffer.ToArray()); + + string stringResult = Encoding.UTF8.GetString(result.Buffer.ToArray()); + pipe.Reader.AdvanceTo(result.Buffer.End); + return stringResult; } public override async Task SerializeWrapper(object value, JsonTypeInfo jsonTypeInfo) @@ -952,7 +964,10 @@ public override async Task SerializeWrapper(object value, JsonTypeInfo j Pipe pipe = new Pipe(); await JsonSerializer.SerializeAsync(pipe.Writer, value, jsonTypeInfo); ReadResult result = await pipe.Reader.ReadAsync(); - return Encoding.UTF8.GetString(result.Buffer.ToArray()); + + string stringResult = Encoding.UTF8.GetString(result.Buffer.ToArray()); + pipe.Reader.AdvanceTo(result.Buffer.End); + return stringResult; } } } From 9448b38db216fb04126836fb7319d5c4e307667f Mon Sep 17 00:00:00 2001 From: Brennan Date: Tue, 23 Apr 2024 21:15:37 -0700 Subject: [PATCH 07/14] system.buffer is special --- src/libraries/System.IO.Pipelines/src/System.IO.Pipelines.csproj | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libraries/System.IO.Pipelines/src/System.IO.Pipelines.csproj b/src/libraries/System.IO.Pipelines/src/System.IO.Pipelines.csproj index c505f82d7e45f..a4517443d1832 100644 --- a/src/libraries/System.IO.Pipelines/src/System.IO.Pipelines.csproj +++ b/src/libraries/System.IO.Pipelines/src/System.IO.Pipelines.csproj @@ -66,7 +66,6 @@ System.IO.Pipelines.PipeReader - From eb2c628911bc0fa6a9dff89d7695b840f65bbf69 Mon Sep 17 00:00:00 2001 From: Brennan Date: Thu, 25 Apr 2024 20:37:40 -0700 Subject: [PATCH 08/14] tests --- .../Metadata/JsonTypeInfoOfT.WriteHelpers.cs | 21 +- .../Serialization/CollectionTests.cs | 7 +- .../Serialization/ConstructorTests.cs | 8 +- .../Serialization/InvalidTypeTests.cs | 7 +- .../JsonCreationHandlingTests.cs | 7 +- .../JsonSerializerApiValidation.cs | 7 +- .../JsonSerializerWrapper.Reflection.cs | 23 +- .../MetadataTests/MetadataTests.cs | 7 +- .../Serialization/NumberHandlingTests.cs | 7 +- .../Serialization/Pipe.WriteTests.cs | 417 ++++++++++++++++++ .../Serialization/PolymorphicTests.cs | 7 +- .../Serialization/ReferenceHandlerTests.cs | 7 +- .../Serialization/RequiredKeywordTests.cs | 7 +- .../UnmappedMemberHandlingTests.cs | 7 +- .../System.Text.Json.Tests.csproj | 1 + 15 files changed, 514 insertions(+), 26 deletions(-) create mode 100644 src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Pipe.WriteTests.cs diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs index 6a31f3bd1983c..74b2d2646a9c7 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs @@ -77,7 +77,7 @@ internal async Task SerializeAsync( Debug.Assert(CanUseSerializeHandler); Debug.Assert(Converter is JsonMetadataServicesConverter); - using var bufferWriter = new PooledByteBufferWriter(Options.DefaultBufferSize); + using PooledByteBufferWriter? bufferWriter = new PooledByteBufferWriter(Options.DefaultBufferSize); Utf8JsonWriter writer = Utf8JsonWriterCache.RentWriter(Options, bufferWriter); try @@ -118,8 +118,8 @@ rootValue is not null && state.CancellationToken = cancellationToken; - using var bufferWriter = new PooledByteBufferWriter(Options.DefaultBufferSize); - using var writer = new Utf8JsonWriter(bufferWriter, Options.GetWriterOptions()); + using PooledByteBufferWriter bufferWriter = new PooledByteBufferWriter(Options.DefaultBufferSize); + using Utf8JsonWriter writer = new Utf8JsonWriter(bufferWriter, Options.GetWriterOptions()); try { @@ -245,14 +245,13 @@ rootValue is not null && state.CancellationToken = cancellationToken; - using var writer = new Utf8JsonWriter(utf8Json, Options.GetWriterOptions()); + using Utf8JsonWriter writer = new Utf8JsonWriter(utf8Json, Options.GetWriterOptions()); try { do { - // Arbitrary threshold, default Pipe buffers are 4k so this allows 4 buffers to be created and filled (mostly) before flushing. - state.FlushThreshold = (int)(16384 * JsonSerializer.FlushThreshold); + state.FlushThreshold = (int)((4 * PipeOptions.Default.MinimumSegmentSize) * JsonSerializer.FlushThreshold); try { @@ -267,7 +266,11 @@ rootValue is not null && } else { - await utf8Json.FlushAsync(cancellationToken).ConfigureAwait(false); + FlushResult result = await utf8Json.FlushAsync(cancellationToken).ConfigureAwait(false); + if (result.IsCanceled || result.IsCompleted) + { + break; + } } } finally @@ -368,8 +371,8 @@ rootValue is not null && supportContinuation: true, supportAsync: false); - using var bufferWriter = new PooledByteBufferWriter(Options.DefaultBufferSize); - using var writer = new Utf8JsonWriter(bufferWriter, Options.GetWriterOptions()); + using PooledByteBufferWriter bufferWriter = new PooledByteBufferWriter(Options.DefaultBufferSize); + using Utf8JsonWriter writer = new Utf8JsonWriter(bufferWriter, Options.GetWriterOptions()); do { diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/CollectionTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/CollectionTests.cs index 6589f6cc9edbb..f8050941dadb2 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/CollectionTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/CollectionTests.cs @@ -27,6 +27,11 @@ public CollectionTestsDynamic_SyncStream() : base(JsonSerializerWrapper.SyncStre public sealed partial class CollectionTestsDynamic_Pipe : CollectionTests { - public CollectionTestsDynamic_Pipe() : base(JsonSerializerWrapper.PipeSerializer) { } + public CollectionTestsDynamic_Pipe() : base(JsonSerializerWrapper.AsyncPipeSerializer) { } + } + + public sealed partial class CollectionTestsDynamic_PipeWithSmallBuffer : CollectionTests + { + public CollectionTestsDynamic_PipeWithSmallBuffer() : base(JsonSerializerWrapper.AsyncPipeSerializerWithSmallBuffer) { } } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ConstructorTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ConstructorTests.cs index 202bcab22599a..914797b976dc0 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ConstructorTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ConstructorTests.cs @@ -31,6 +31,12 @@ public ConstructorTests_Span() public class ConstructorTests_Pipe : ConstructorTests { public ConstructorTests_Pipe() - : base(JsonSerializerWrapper.PipeSerializer) { } + : base(JsonSerializerWrapper.AsyncPipeSerializer) { } + } + + public class ConstructorTests_PipeWithSmallBuffer : ConstructorTests + { + public ConstructorTests_PipeWithSmallBuffer() + : base(JsonSerializerWrapper.AsyncPipeSerializerWithSmallBuffer) { } } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/InvalidTypeTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/InvalidTypeTests.cs index e09e4a00c8e2a..4b8be091ccb95 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/InvalidTypeTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/InvalidTypeTests.cs @@ -39,7 +39,12 @@ public InvalidTypeTests_Writer() : base(JsonSerializerWrapper.ReaderWriterSerial public class InvalidTypeTests_Pipe : InvalidTypeTests { - public InvalidTypeTests_Pipe() : base(JsonSerializerWrapper.PipeSerializer) { } + public InvalidTypeTests_Pipe() : base(JsonSerializerWrapper.AsyncPipeSerializer) { } + } + + public class InvalidTypeTests_PipeWithSmallBuffer : InvalidTypeTests + { + public InvalidTypeTests_PipeWithSmallBuffer() : base(JsonSerializerWrapper.AsyncPipeSerializerWithSmallBuffer) { } } public abstract class InvalidTypeTests diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonCreationHandlingTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonCreationHandlingTests.cs index 7cd83ab63944a..b901cd079599c 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonCreationHandlingTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonCreationHandlingTests.cs @@ -25,6 +25,11 @@ public JsonCreationHandlingTests_SyncStream() : base(JsonSerializerWrapper.SyncS public sealed class JsonCreationHandlingTests_Pipe : JsonCreationHandlingTests { - public JsonCreationHandlingTests_Pipe() : base(JsonSerializerWrapper.PipeSerializer) { } + public JsonCreationHandlingTests_Pipe() : base(JsonSerializerWrapper.AsyncPipeSerializer) { } + } + + public sealed class JsonCreationHandlingTests_PipeWithSmallBuffer : JsonCreationHandlingTests + { + public JsonCreationHandlingTests_PipeWithSmallBuffer() : base(JsonSerializerWrapper.AsyncPipeSerializerWithSmallBuffer) { } } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerApiValidation.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerApiValidation.cs index d3353e5364bfa..587893e7392b7 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerApiValidation.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerApiValidation.cs @@ -53,7 +53,12 @@ public JsonSerializerApiValidation_Node() : base(JsonSerializerWrapper.NodeSeria public class JsonSerializerApiValidation_Pipe : JsonSerializerApiValidation { - public JsonSerializerApiValidation_Pipe() : base(JsonSerializerWrapper.PipeSerializer) { } + public JsonSerializerApiValidation_Pipe() : base(JsonSerializerWrapper.AsyncPipeSerializer) { } + } + + public class JsonSerializerApiValidation_PipeWithSmallBuffer : JsonSerializerApiValidation + { + public JsonSerializerApiValidation_PipeWithSmallBuffer() : base(JsonSerializerWrapper.AsyncPipeSerializerWithSmallBuffer) { } } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerWrapper.Reflection.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerWrapper.Reflection.cs index 6282c24e38249..af436e96b358a 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerWrapper.Reflection.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerWrapper.Reflection.cs @@ -34,7 +34,8 @@ protected JsonSerializerWrapper() public static JsonSerializerWrapper DocumentSerializer { get; } = new DocumentSerializerWrapper(); public static JsonSerializerWrapper ElementSerializer { get; } = new ElementSerializerWrapper(); public static JsonSerializerWrapper NodeSerializer { get; } = new NodeSerializerWrapper(); - public static JsonSerializerWrapper PipeSerializer { get; } = new PipelinesSerializerWrapper(); + public static JsonSerializerWrapper AsyncPipeSerializer { get; } = new AsyncPipelinesSerializerWrapper(); + public static JsonSerializerWrapper AsyncPipeSerializerWithSmallBuffer { get; } = new AsyncPipelinesSerializerWrapper(forceSmallBufferInOptions: true); private class SpanSerializerWrapper : JsonSerializerWrapper { @@ -886,18 +887,28 @@ private int ReadExactlyFromSource(byte[] buffer, int offset, int count) } // TODO: Deserialize to use PipeReader overloads once implemented - private class PipelinesSerializerWrapper : JsonSerializerWrapper + private class AsyncPipelinesSerializerWrapper : JsonSerializerWrapper { + private readonly bool _forceSmallBufferInOptions; + public override JsonSerializerOptions DefaultOptions => JsonSerializerOptions.Default; public override bool SupportsNullValueOnDeserialize => true; + private JsonSerializerOptions? ResolveOptionsInstance(JsonSerializerOptions? options) + => _forceSmallBufferInOptions ? JsonSerializerOptionsSmallBufferMapper.ResolveOptionsInstanceWithSmallBuffer(options) : options; + + public AsyncPipelinesSerializerWrapper(bool forceSmallBufferInOptions = false) + { + _forceSmallBufferInOptions = forceSmallBufferInOptions; + } + public override async Task DeserializeWrapper(string json, JsonSerializerOptions options = null) { - return await JsonSerializer.DeserializeAsync(new MemoryStream(Encoding.UTF8.GetBytes(json)), options); + return await JsonSerializer.DeserializeAsync(new MemoryStream(Encoding.UTF8.GetBytes(json)), ResolveOptionsInstance(options)); } public override async Task DeserializeWrapper(string json, Type type, JsonSerializerOptions options = null) { - return await JsonSerializer.DeserializeAsync(new MemoryStream(Encoding.UTF8.GetBytes(json)), type, options); + return await JsonSerializer.DeserializeAsync(new MemoryStream(Encoding.UTF8.GetBytes(json)), type, ResolveOptionsInstance(options)); } public override async Task DeserializeWrapper(string json, JsonTypeInfo jsonTypeInfo) @@ -918,7 +929,7 @@ public override async Task DeserializeWrapper(string json, Type type, Js public override async Task SerializeWrapper(object value, Type inputType, JsonSerializerOptions options = null) { Pipe pipe = new Pipe(); - await JsonSerializer.SerializeAsync(pipe.Writer, value, inputType, options); + await JsonSerializer.SerializeAsync(pipe.Writer, value, inputType, ResolveOptionsInstance(options)); ReadResult result = await pipe.Reader.ReadAsync(); string stringResult = Encoding.UTF8.GetString(result.Buffer.ToArray()); @@ -929,7 +940,7 @@ public override async Task SerializeWrapper(object value, Type inputType public override async Task SerializeWrapper(T value, JsonSerializerOptions options = null) { Pipe pipe = new Pipe(); - await JsonSerializer.SerializeAsync(pipe.Writer, value, options); + await JsonSerializer.SerializeAsync(pipe.Writer, value, ResolveOptionsInstance(options)); ReadResult result = await pipe.Reader.ReadAsync(); string stringResult = Encoding.UTF8.GetString(result.Buffer.ToArray()); diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/MetadataTests/MetadataTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/MetadataTests/MetadataTests.cs index a9e943ff06856..ea26ca1f91106 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/MetadataTests/MetadataTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/MetadataTests/MetadataTests.cs @@ -49,7 +49,12 @@ public MetadataTests_Node() : base(JsonSerializerWrapper.NodeSerializer) { } public class MetadataTests_Pipe : MetadataTests { - public MetadataTests_Pipe() : base(JsonSerializerWrapper.PipeSerializer) { } + public MetadataTests_Pipe() : base(JsonSerializerWrapper.AsyncPipeSerializer) { } + } + + public class MetadataTests_PipeWithSmallBuffer : MetadataTests + { + public MetadataTests_PipeWithSmallBuffer() : base(JsonSerializerWrapper.AsyncPipeSerializerWithSmallBuffer) { } } public abstract partial class MetadataTests diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/NumberHandlingTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/NumberHandlingTests.cs index f1229f2c7e986..b5cb5381b07f3 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/NumberHandlingTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/NumberHandlingTests.cs @@ -44,7 +44,12 @@ public NumberHandlingTests_Node() : base(JsonSerializerWrapper.NodeSerializer) { public class NumberHandlingTests_Pipe : NumberHandlingTests_OverloadSpecific { - public NumberHandlingTests_Pipe() : base(JsonSerializerWrapper.PipeSerializer) { } + public NumberHandlingTests_Pipe() : base(JsonSerializerWrapper.AsyncPipeSerializer) { } + } + + public class NumberHandlingTests_PipeWithSmallBuffer : NumberHandlingTests_OverloadSpecific + { + public NumberHandlingTests_PipeWithSmallBuffer() : base(JsonSerializerWrapper.AsyncPipeSerializerWithSmallBuffer) { } } public abstract class NumberHandlingTests_OverloadSpecific diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Pipe.WriteTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Pipe.WriteTests.cs new file mode 100644 index 0000000000000..bed9e9c2aff65 --- /dev/null +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Pipe.WriteTests.cs @@ -0,0 +1,417 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers; +using System.Collections.Generic; +using System.IO.Pipelines; +using System.Runtime.CompilerServices; +using System.Text.Json.Serialization.Metadata; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace System.Text.Json.Serialization.Tests +{ + public partial class PipeTests + { + [Fact] + public async Task WriteNullArgumentFail() + { + await Assert.ThrowsAsync(async () => await JsonSerializer.SerializeAsync((PipeWriter)null, 1)); + await Assert.ThrowsAsync(async () => await JsonSerializer.SerializeAsync((PipeWriter)null, 1, typeof(int))); + } + + [Fact] + public async Task VerifyValueFail() + { + Pipe pipe = new Pipe(); + await Assert.ThrowsAsync(async () => await JsonSerializer.SerializeAsync(pipe.Writer, "", (Type)null)); + } + + [Fact] + public async Task VerifyTypeFail() + { + Pipe pipe = new Pipe(); + await Assert.ThrowsAsync(async () => await JsonSerializer.SerializeAsync(pipe.Writer, 1, typeof(string))); + } + + [Fact] + public async Task CompletedPipeWithExceptionThrowsFromSerialize() + { + Pipe pipe = new Pipe(); + pipe.Reader.Complete(new Exception()); + + await Assert.ThrowsAsync(() => JsonSerializer.SerializeAsync(pipe.Writer, 1)); + } + + [Fact] + public async Task CancelPendingFlushDuringBackpressureReturns() + { + Pipe pipe = new Pipe(new PipeOptions(pauseWriterThreshold: 10, resumeWriterThreshold: 5)); + await pipe.Writer.WriteAsync("123456789"u8.ToArray()); + Task serializeTask = JsonSerializer.SerializeAsync(pipe.Writer, GetNumbersAsync()); + Assert.False(serializeTask.IsCompleted); + + pipe.Writer.CancelPendingFlush(); + + await serializeTask; + + ReadResult result = await pipe.Reader.ReadAsync(); + // Buffer: 012345679[0 + Assert.Equal(11, result.Buffer.Length); + pipe.Reader.AdvanceTo(result.Buffer.End); + + static async IAsyncEnumerable GetNumbersAsync() + { + int i = 0; + while (true) + { + await Task.Delay(10); + yield return i++; + } + } + } + + [Fact] + public async Task BackpressureIsObservedWhenWritingJson() + { + Pipe pipe = new Pipe(new PipeOptions(pauseWriterThreshold: 10, resumeWriterThreshold: 5)); + await pipe.Writer.WriteAsync("123456789"u8.ToArray()); + Task serializeTask = JsonSerializer.SerializeAsync(pipe.Writer, 1); + Assert.False(serializeTask.IsCompleted); + + ReadResult result = await pipe.Reader.ReadAsync(); + pipe.Reader.AdvanceTo(result.Buffer.GetPosition(5)); + + // Still need to read 1 more byte to unblock flush + Assert.False(serializeTask.IsCompleted); + + result = await pipe.Reader.ReadAsync(); + pipe.Reader.AdvanceTo(result.Buffer.GetPosition(1)); + + await serializeTask; + } + + [Fact] + public async Task CanCancelBackpressuredJsonWrite() + { + Pipe pipe = new Pipe(new PipeOptions(pauseWriterThreshold: 10, resumeWriterThreshold: 5)); + await pipe.Writer.WriteAsync("123456789"u8.ToArray()); + + CancellationTokenSource cts = new(); + Task serializeTask = JsonSerializer.SerializeAsync(pipe.Writer, 1, cancellationToken: cts.Token); + Assert.False(serializeTask.IsCompleted); + + cts.Cancel(); + + await Assert.ThrowsAsync(() => serializeTask); + + ReadResult result = await pipe.Reader.ReadAsync(); + // Even though flush was canceled, the bytes are still written to the Pipe + Assert.Equal(10, result.Buffer.Length); + pipe.Reader.AdvanceTo(result.Buffer.End); + } + + [Theory] + [InlineData(32)] + [InlineData(128)] + [InlineData(1024)] + [InlineData(1024 * 16)] // the default JsonSerializerOptions.DefaultBufferSize value + [InlineData(1024 * 1024)] + public async Task ShouldUseFastPathOnSmallPayloads(int defaultBufferSize) + { + var instrumentedResolver = new PocoWithInstrumentedFastPath.Context( + new JsonSerializerOptions + { + DefaultBufferSize = defaultBufferSize, + }); + + // The current implementation uses a heuristic + int smallValueThreshold = defaultBufferSize / 2; + PocoWithInstrumentedFastPath smallValue = PocoWithInstrumentedFastPath.CreateValueWithSerializationSize(smallValueThreshold); + + // We don't care about backpressure in this test + Pipe pipe = new Pipe(new PipeOptions(pauseWriterThreshold: defaultBufferSize, resumeWriterThreshold: defaultBufferSize / 2)); + ReadResult result; + + // The first 10 serializations should not call into the fast path + for (int i = 0; i < 10; i++) + { + await JsonSerializer.SerializeAsync(pipe.Writer, smallValue, instrumentedResolver.Options); + result = await pipe.Reader.ReadAsync(); + pipe.Reader.AdvanceTo(result.Buffer.End); + Assert.Equal(0, instrumentedResolver.FastPathInvocationCount); + } + + // Subsequent iterations do call into the fast path + for (int i = 0; i < 10; i++) + { + await JsonSerializer.SerializeAsync(pipe.Writer, smallValue, instrumentedResolver.Options); + result = await pipe.Reader.ReadAsync(); + pipe.Reader.AdvanceTo(result.Buffer.End); + Assert.Equal(i + 1, instrumentedResolver.FastPathInvocationCount); + } + + // Polymorphic serialization should use the fast path + await JsonSerializer.SerializeAsync(pipe.Writer, (object)smallValue, instrumentedResolver.Options); + result = await pipe.Reader.ReadAsync(); + pipe.Reader.AdvanceTo(result.Buffer.End); + Assert.Equal(11, instrumentedResolver.FastPathInvocationCount); + + // Attempt to serialize a value that is deemed large + var largeValue = PocoWithInstrumentedFastPath.CreateValueWithSerializationSize(smallValueThreshold + 1); + await JsonSerializer.SerializeAsync(pipe.Writer, largeValue, instrumentedResolver.Options); + result = await pipe.Reader.ReadAsync(); + pipe.Reader.AdvanceTo(result.Buffer.End); + Assert.Equal(12, instrumentedResolver.FastPathInvocationCount); + + // Any subsequent attempts no longer call into the fast path + for (int i = 0; i < 10; i++) + { + await JsonSerializer.SerializeAsync(pipe.Writer, smallValue, instrumentedResolver.Options); + result = await pipe.Reader.ReadAsync(); + pipe.Reader.AdvanceTo(result.Buffer.End); + Assert.Equal(12, instrumentedResolver.FastPathInvocationCount); + } + } + + [Fact] + public async Task FastPathObservesBackpressure() + { + int defaultBufferSize = 4096; + var instrumentedResolver = new PocoWithInstrumentedFastPath.Context( + new JsonSerializerOptions + { + DefaultBufferSize = defaultBufferSize, + }); + + // The current implementation uses a heuristic + int smallValueThreshold = defaultBufferSize / 2; + PocoWithInstrumentedFastPath smallValue = PocoWithInstrumentedFastPath.CreateValueWithSerializationSize(smallValueThreshold); + + Pipe pipe = new Pipe(new PipeOptions(pauseWriterThreshold: defaultBufferSize / 2, resumeWriterThreshold: defaultBufferSize / 4)); + ReadResult result; + + // The first 10 serializations should not call into the fast path + for (int i = 0; i < 10; i++) + { + await JsonSerializer.SerializeAsync(pipe.Writer, smallValue, instrumentedResolver.Options); + result = await pipe.Reader.ReadAsync(); + pipe.Reader.AdvanceTo(result.Buffer.End); + Assert.Equal(0, instrumentedResolver.FastPathInvocationCount); + } + + Task serializeTask = JsonSerializer.SerializeAsync(pipe.Writer, smallValue, instrumentedResolver.Options); + Assert.False(serializeTask.IsCompleted); + Assert.Equal(1, instrumentedResolver.FastPathInvocationCount); + + result = await pipe.Reader.ReadAsync(); + pipe.Reader.AdvanceTo(result.Buffer.End); + await serializeTask; + } + + [Fact] + public async Task CanCancelBackpressuredFastPath() + { + int defaultBufferSize = 4096; + var instrumentedResolver = new PocoWithInstrumentedFastPath.Context( + new JsonSerializerOptions + { + DefaultBufferSize = defaultBufferSize, + }); + + // The current implementation uses a heuristic + int smallValueThreshold = defaultBufferSize / 2; + PocoWithInstrumentedFastPath smallValue = PocoWithInstrumentedFastPath.CreateValueWithSerializationSize(smallValueThreshold); + + Pipe pipe = new Pipe(new PipeOptions(pauseWriterThreshold: defaultBufferSize / 2, resumeWriterThreshold: defaultBufferSize / 4)); + ReadResult result; + + // The first 10 serializations should not call into the fast path + for (int i = 0; i < 10; i++) + { + await JsonSerializer.SerializeAsync(pipe.Writer, smallValue, instrumentedResolver.Options); + result = await pipe.Reader.ReadAsync(); + pipe.Reader.AdvanceTo(result.Buffer.End); + Assert.Equal(0, instrumentedResolver.FastPathInvocationCount); + } + + CancellationTokenSource cts = new(); + + Task serializeTask = JsonSerializer.SerializeAsync(pipe.Writer, smallValue, instrumentedResolver.Options, cts.Token); + Assert.False(serializeTask.IsCompleted); + + cts.Cancel(); + await Assert.ThrowsAsync(() => serializeTask); + } + + [Fact] + public async Task BuffersBehaveAsExpected() + { + TestPool pool = new TestPool(2000); + Pipe pipe = new Pipe(new PipeOptions(pool)); + + // Many small writes + for (int i = 0; i < 100; ++i) + { + await JsonSerializer.SerializeAsync(pipe.Writer, "a"); + } + // Should fit into a single 2000 byte buffer + Assert.Equal(1, pool.BufferCount); + ReadResult result = await pipe.Reader.ReadAsync(); + pipe.Reader.AdvanceTo(result.Buffer.End); + Assert.Equal(0, pool.BufferCount); + + // Partially fill Pipe so next write needs a new buffer + await JsonSerializer.SerializeAsync(pipe.Writer, new string('a', 600)); + Assert.Equal(1, pool.BufferCount); + + // Writing strings incurs a 3x buffer size due to max potential transcoding + // 600 + 600*3 > 2000 means a second buffer will be grabbed + await JsonSerializer.SerializeAsync(pipe.Writer, new string('a', 600)); + Assert.Equal(2, pool.BufferCount); + result = await pipe.Reader.ReadAsync(); + Assert.Equal(1204, result.Buffer.Length); + SequencePosition pos = result.Buffer.Start; + int segments = 0; + while (result.Buffer.TryGet(ref pos, out ReadOnlyMemory memory)) + { + segments++; + Assert.Equal(602, memory.Length); + } + Assert.Equal(2, segments); + pipe.Reader.AdvanceTo(result.Buffer.End); + + // Large write + await JsonSerializer.SerializeAsync(pipe.Writer, new string('a', 2000)); + // Write is larger than pools max buffer size so Pipes will provide a buffer from elsewhere. + Assert.Equal(0, pool.BufferCount); + result = await pipe.Reader.ReadAsync(); + Assert.Equal(2002, result.Buffer.Length); + + //pipe.Writer.CancelPendingFlush(); + } + + internal class TestPool : MemoryPool + { + private readonly int _bufferSize; + private int _bufferCount; + + public int BufferCount => _bufferCount; + + public TestPool(int bufferSize) + { + _bufferSize = bufferSize; + } + + public override int MaxBufferSize => _bufferSize; + + public override IMemoryOwner Rent(int minBufferSize = -1) + { + _bufferCount++; + return new MemoryPoolBuffer(_bufferSize, this); + } + + protected override void Dispose(bool disposing) + { } + + private sealed class MemoryPoolBuffer : IMemoryOwner + { + private readonly TestPool _pool; + private byte[]? _array; + private int _length; + + public MemoryPoolBuffer(int size, TestPool pool) + { + _array = ArrayPool.Shared.Rent(size); + _length = size; + _pool = pool; + } + + public Memory Memory + { + get + { + byte[]? array = _array; + + return new Memory(array, 0, _length); + } + } + + public void Dispose() + { + byte[]? array = _array; + if (array != null) + { + _array = null; + ArrayPool.Shared.Return(array); + _pool._bufferCount--; + } + } + } + } + + internal class PocoWithInstrumentedFastPath + { + public static PocoWithInstrumentedFastPath CreateValueWithSerializationSize(int targetSerializationSize) + { + int objectSerializationPaddingSize = """{"Value":""}""".Length; // 12 + return new PocoWithInstrumentedFastPath { Value = new string('a', targetSerializationSize - objectSerializationPaddingSize) }; + } + + public string? Value { get; set; } + + public class Context : JsonSerializerContext, IJsonTypeInfoResolver + { + public int FastPathInvocationCount { get; private set; } + + public Context(JsonSerializerOptions options) : base(options) + { } + + protected override JsonSerializerOptions? GeneratedSerializerOptions => Options; + public override JsonTypeInfo? GetTypeInfo(Type type) => GetTypeInfo(type, Options); + + public JsonTypeInfo? GetTypeInfo(Type type, JsonSerializerOptions options) + { + if (type == typeof(string)) + { + return JsonMetadataServices.CreateValueInfo(options, JsonMetadataServices.StringConverter); + } + + if (type == typeof(object)) + { + return JsonMetadataServices.CreateValueInfo(options, JsonMetadataServices.ObjectConverter); + } + + if (type == typeof(PocoWithInstrumentedFastPath)) + { + return JsonMetadataServices.CreateObjectInfo(options, + new JsonObjectInfoValues + { + PropertyMetadataInitializer = _ => new JsonPropertyInfo[1] + { + JsonMetadataServices.CreatePropertyInfo(options, + new JsonPropertyInfoValues + { + DeclaringType = typeof(PocoWithInstrumentedFastPath), + PropertyName = "Value", + Getter = obj => ((PocoWithInstrumentedFastPath)obj).Value, + }) + }, + + SerializeHandler = (writer, value) => + { + writer.WriteStartObject(); + writer.WriteString("Value", value.Value); + writer.WriteEndObject(); + FastPathInvocationCount++; + } + }); + } + + return null; + } + } + } + } +} diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/PolymorphicTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/PolymorphicTests.cs index 9938a949e2dce..2ce3fdbc39aee 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/PolymorphicTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/PolymorphicTests.cs @@ -57,7 +57,12 @@ public PolymorphicTests_Node() : base(JsonSerializerWrapper.NodeSerializer) { } public class PolymorphicTests_Pipe : PolymorphicTests { - public PolymorphicTests_Pipe() : base(JsonSerializerWrapper.PipeSerializer) { } + public PolymorphicTests_Pipe() : base(JsonSerializerWrapper.AsyncPipeSerializer) { } + } + + public class PolymorphicTests_PipeWithSmallBuffer : PolymorphicTests + { + public PolymorphicTests_PipeWithSmallBuffer() : base(JsonSerializerWrapper.AsyncPipeSerializerWithSmallBuffer) { } } public abstract partial class PolymorphicTests : SerializerTests diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ReferenceHandlerTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ReferenceHandlerTests.cs index 21781b06f6660..366d9c0d84040 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ReferenceHandlerTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ReferenceHandlerTests.cs @@ -32,6 +32,11 @@ public ReferenceHandlerTestsDynamic_IgnoreCycles_AsyncStream() : base(JsonSerial public sealed class ReferenceHandlerTestsDynamic_IgnoreCycles_Pipe : ReferenceHandlerTests_IgnoreCycles { - public ReferenceHandlerTestsDynamic_IgnoreCycles_Pipe() : base(JsonSerializerWrapper.PipeSerializer) { } + public ReferenceHandlerTestsDynamic_IgnoreCycles_Pipe() : base(JsonSerializerWrapper.AsyncPipeSerializer) { } + } + + public sealed class ReferenceHandlerTestsDynamic_IgnoreCycles_PipeWithSmallBuffer : ReferenceHandlerTests_IgnoreCycles + { + public ReferenceHandlerTestsDynamic_IgnoreCycles_PipeWithSmallBuffer() : base(JsonSerializerWrapper.AsyncPipeSerializerWithSmallBuffer) { } } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/RequiredKeywordTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/RequiredKeywordTests.cs index ce78182ee3262..42dfbe331f860 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/RequiredKeywordTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/RequiredKeywordTests.cs @@ -55,6 +55,11 @@ public RequiredKeywordTests_Node() : base(JsonSerializerWrapper.NodeSerializer) public class RequiredKeywordTests_Pipe : RequiredKeywordTests { - public RequiredKeywordTests_Pipe() : base(JsonSerializerWrapper.PipeSerializer) { } + public RequiredKeywordTests_Pipe() : base(JsonSerializerWrapper.AsyncPipeSerializer) { } + } + + public class RequiredKeywordTests_PipeWithSmallBuffer : RequiredKeywordTests + { + public RequiredKeywordTests_PipeWithSmallBuffer() : base(JsonSerializerWrapper.AsyncPipeSerializerWithSmallBuffer) { } } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/UnmappedMemberHandlingTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/UnmappedMemberHandlingTests.cs index 8434b3b8444f2..7b3dac7983df1 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/UnmappedMemberHandlingTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/UnmappedMemberHandlingTests.cs @@ -15,6 +15,11 @@ public UnmappedMemberHandlingTests_AsyncStream() : base(JsonSerializerWrapper.As public sealed partial class UnmappedMemberHandlingTests_Pipe : UnmappedMemberHandlingTests { - public UnmappedMemberHandlingTests_Pipe() : base(JsonSerializerWrapper.PipeSerializer) { } + public UnmappedMemberHandlingTests_Pipe() : base(JsonSerializerWrapper.AsyncPipeSerializer) { } + } + + public sealed partial class UnmappedMemberHandlingTests_PipeWithSmallBuffer : UnmappedMemberHandlingTests + { + public UnmappedMemberHandlingTests_PipeWithSmallBuffer() : base(JsonSerializerWrapper.AsyncPipeSerializerWithSmallBuffer) { } } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj index b3a12d41a753b..acfce7b96e177 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj @@ -165,6 +165,7 @@ + From dde0db0723e5b498c459198a9448e22a278f846a Mon Sep 17 00:00:00 2001 From: Brennan Date: Thu, 2 May 2024 16:16:07 -0700 Subject: [PATCH 09/14] fixup --- .../Serialization/ConstructorTests.cs | 6 ------ .../Serialization/JsonSerializerApiValidation.cs | 5 ----- .../Serialization/MetadataTests/MetadataTests.cs | 5 ----- .../Serialization/NumberHandlingTests.cs | 5 ----- .../System.Text.Json.Tests/Serialization/Pipe.WriteTests.cs | 2 +- .../Serialization/UnmappedMemberHandlingTests.cs | 5 ----- 6 files changed, 1 insertion(+), 27 deletions(-) diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ConstructorTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ConstructorTests.cs index 914797b976dc0..ca5c31afc1b81 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ConstructorTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ConstructorTests.cs @@ -33,10 +33,4 @@ public class ConstructorTests_Pipe : ConstructorTests public ConstructorTests_Pipe() : base(JsonSerializerWrapper.AsyncPipeSerializer) { } } - - public class ConstructorTests_PipeWithSmallBuffer : ConstructorTests - { - public ConstructorTests_PipeWithSmallBuffer() - : base(JsonSerializerWrapper.AsyncPipeSerializerWithSmallBuffer) { } - } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerApiValidation.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerApiValidation.cs index 587893e7392b7..1ed08df56d62c 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerApiValidation.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerApiValidation.cs @@ -55,11 +55,6 @@ public class JsonSerializerApiValidation_Pipe : JsonSerializerApiValidation { public JsonSerializerApiValidation_Pipe() : base(JsonSerializerWrapper.AsyncPipeSerializer) { } } - - public class JsonSerializerApiValidation_PipeWithSmallBuffer : JsonSerializerApiValidation - { - public JsonSerializerApiValidation_PipeWithSmallBuffer() : base(JsonSerializerWrapper.AsyncPipeSerializerWithSmallBuffer) { } - } } /// diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/MetadataTests/MetadataTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/MetadataTests/MetadataTests.cs index ea26ca1f91106..7d8c2238a085d 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/MetadataTests/MetadataTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/MetadataTests/MetadataTests.cs @@ -52,11 +52,6 @@ public class MetadataTests_Pipe : MetadataTests public MetadataTests_Pipe() : base(JsonSerializerWrapper.AsyncPipeSerializer) { } } - public class MetadataTests_PipeWithSmallBuffer : MetadataTests - { - public MetadataTests_PipeWithSmallBuffer() : base(JsonSerializerWrapper.AsyncPipeSerializerWithSmallBuffer) { } - } - public abstract partial class MetadataTests { protected JsonSerializerWrapper Serializer { get; } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/NumberHandlingTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/NumberHandlingTests.cs index b5cb5381b07f3..5613dda779335 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/NumberHandlingTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/NumberHandlingTests.cs @@ -47,11 +47,6 @@ public class NumberHandlingTests_Pipe : NumberHandlingTests_OverloadSpecific public NumberHandlingTests_Pipe() : base(JsonSerializerWrapper.AsyncPipeSerializer) { } } - public class NumberHandlingTests_PipeWithSmallBuffer : NumberHandlingTests_OverloadSpecific - { - public NumberHandlingTests_PipeWithSmallBuffer() : base(JsonSerializerWrapper.AsyncPipeSerializerWithSmallBuffer) { } - } - public abstract class NumberHandlingTests_OverloadSpecific { private JsonSerializerWrapper Serializer { get; } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Pipe.WriteTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Pipe.WriteTests.cs index bed9e9c2aff65..ceffcb392d6b2 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Pipe.WriteTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Pipe.WriteTests.cs @@ -57,7 +57,7 @@ public async Task CancelPendingFlushDuringBackpressureReturns() await serializeTask; ReadResult result = await pipe.Reader.ReadAsync(); - // Buffer: 012345679[0 + // result.Buffer: 123456789[0 Assert.Equal(11, result.Buffer.Length); pipe.Reader.AdvanceTo(result.Buffer.End); diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/UnmappedMemberHandlingTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/UnmappedMemberHandlingTests.cs index 7b3dac7983df1..5c59342f3c779 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/UnmappedMemberHandlingTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/UnmappedMemberHandlingTests.cs @@ -17,9 +17,4 @@ public sealed partial class UnmappedMemberHandlingTests_Pipe : UnmappedMemberHan { public UnmappedMemberHandlingTests_Pipe() : base(JsonSerializerWrapper.AsyncPipeSerializer) { } } - - public sealed partial class UnmappedMemberHandlingTests_PipeWithSmallBuffer : UnmappedMemberHandlingTests - { - public UnmappedMemberHandlingTests_PipeWithSmallBuffer() : base(JsonSerializerWrapper.AsyncPipeSerializerWithSmallBuffer) { } - } } From 2c18119770605b8a512ecf0a9cdb4ad4c9d4090d Mon Sep 17 00:00:00 2001 From: Brennan Date: Wed, 8 May 2024 15:22:36 -0700 Subject: [PATCH 10/14] de-dupe --- .../src/Resources/Strings.resx | 53 ++++-- .../src/System.Text.Json.csproj | 1 + .../AsyncSerializationBufferWriterContext.cs | 83 +++++++++ .../Serialization/Metadata/JsonTypeInfo.cs | 5 +- .../Metadata/JsonTypeInfoOfT.WriteHelpers.cs | 169 ++++-------------- .../Text/Json/ThrowHelper.Serialization.cs | 12 ++ .../Serialization/Pipe.WriteTests.cs | 19 +- 7 files changed, 180 insertions(+), 162 deletions(-) create mode 100644 src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncSerializationBufferWriterContext.cs diff --git a/src/libraries/System.Text.Json/src/Resources/Strings.resx b/src/libraries/System.Text.Json/src/Resources/Strings.resx index acd31cc6cce25..94458aa73e47a 100644 --- a/src/libraries/System.Text.Json/src/Resources/Strings.resx +++ b/src/libraries/System.Text.Json/src/Resources/Strings.resx @@ -1,13 +1,17 @@ - @@ -714,4 +725,10 @@ Indentation size must be between {0} and {1}. - + + PipeWriter.FlushAsync was canceled. + + + PipeWriter has been completed, nothing more can be written to it. + + \ No newline at end of file diff --git a/src/libraries/System.Text.Json/src/System.Text.Json.csproj b/src/libraries/System.Text.Json/src/System.Text.Json.csproj index 15d23ec13533d..a91dcddba93d3 100644 --- a/src/libraries/System.Text.Json/src/System.Text.Json.csproj +++ b/src/libraries/System.Text.Json/src/System.Text.Json.csproj @@ -102,6 +102,7 @@ The System.Text.Json library is built-in as part of the shared framework in .NET + diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncSerializationBufferWriterContext.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncSerializationBufferWriterContext.cs new file mode 100644 index 0000000000000..7cce1e44d6d78 --- /dev/null +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncSerializationBufferWriterContext.cs @@ -0,0 +1,83 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers; +using System.IO; +using System.IO.Pipelines; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +namespace System.Text.Json.Serialization +{ + // Common interface to help de-dupe code for different types that can do async serialization (Stream and PipeWriter) + internal interface IAsyncSerializationBufferWriterContext : IDisposable + { + int FlushThreshold { get; } + + ValueTask FlushAsync(CancellationToken cancellationToken); + + public IBufferWriter BufferWriter { get; } + } + + internal readonly struct AsyncSerializationStreamContext : IAsyncSerializationBufferWriterContext + { + private readonly Stream _stream; + private readonly JsonSerializerOptions _options; + private readonly PooledByteBufferWriter _bufferWriter; + + public AsyncSerializationStreamContext(Stream stream, JsonSerializerOptions options) + { + _stream = stream; + _options = options; + _bufferWriter = new PooledByteBufferWriter(_options.DefaultBufferSize); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public async ValueTask FlushAsync(CancellationToken cancellationToken) + { + await _bufferWriter.WriteToStreamAsync(_stream, cancellationToken).ConfigureAwait(false); + _bufferWriter.Clear(); + } + + public int FlushThreshold => (int)(_options.DefaultBufferSize * JsonSerializer.FlushThreshold); + + public IBufferWriter BufferWriter => _bufferWriter; + + public void Dispose() + { + _bufferWriter.Dispose(); + } + } + + internal readonly struct AsyncSerializationPipeContext : IAsyncSerializationBufferWriterContext + { + private readonly PipeWriter _pipe; + + public AsyncSerializationPipeContext(PipeWriter pipe) + { + _pipe = pipe; + } + + public int FlushThreshold => (int)((4 * PipeOptions.Default.MinimumSegmentSize) * JsonSerializer.FlushThreshold); + + public IBufferWriter BufferWriter => _pipe; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public async ValueTask FlushAsync(CancellationToken cancellationToken) + { + FlushResult result = await _pipe.FlushAsync(cancellationToken).ConfigureAwait(false); + if (result.IsCanceled || result.IsCompleted) + { + if (result.IsCanceled) + { + ThrowHelper.ThrowOperationCanceledException_PipeWriteCanceled(); + } + + ThrowHelper.ThrowOperationCanceledException_PipeWriteCanceled(); + } + } + + public void Dispose() { } + } +} diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfo.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfo.cs index 9afcc553b9ede..02b0d61f0ef6e 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfo.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfo.cs @@ -6,6 +6,7 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.IO; +using System.IO.Pipelines; using System.Runtime.CompilerServices; using System.Runtime.ExceptionServices; using System.Text.Json.Reflection; @@ -989,8 +990,10 @@ public JsonPropertyInfo CreateJsonPropertyInfo(Type propertyType, string name) // Untyped, root-level serialization methods internal abstract void SerializeAsObject(Utf8JsonWriter writer, object? rootValue); + internal abstract Task SerializeAsObjectAsync(TSerializationContext serializationContext, object? rootValue, CancellationToken cancellationToken) + where TSerializationContext : struct, IAsyncSerializationBufferWriterContext; internal abstract Task SerializeAsObjectAsync(Stream utf8Json, object? rootValue, CancellationToken cancellationToken); - internal abstract Task SerializeAsObjectAsync(System.IO.Pipelines.PipeWriter utf8Json, object? rootValue, CancellationToken cancellationToken); + internal abstract Task SerializeAsObjectAsync(PipeWriter utf8Json, object? rootValue, CancellationToken cancellationToken); internal abstract void SerializeAsObject(Stream utf8Json, object? rootValue); // Untyped, root-level deserialization methods diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs index 74b2d2646a9c7..14e5d8bc2f275 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs @@ -59,141 +59,28 @@ rootValue is not null && } } - // Root serialization method for async streaming serialization. - internal async Task SerializeAsync( - Stream utf8Json, + internal Task SerializeAsync(Stream utf8Json, T? rootValue, CancellationToken cancellationToken, object? rootValueBoxed = null) { - Debug.Assert(IsConfigured); - Debug.Assert(rootValueBoxed is null || rootValueBoxed is T); - - if (CanUseSerializeHandlerInStreaming) - { - // Short-circuit calls into SerializeHandler, if the `CanUseSerializeHandlerInStreaming` heuristic allows it. - - Debug.Assert(SerializeHandler != null); - Debug.Assert(CanUseSerializeHandler); - Debug.Assert(Converter is JsonMetadataServicesConverter); - - using PooledByteBufferWriter? bufferWriter = new PooledByteBufferWriter(Options.DefaultBufferSize); - Utf8JsonWriter writer = Utf8JsonWriterCache.RentWriter(Options, bufferWriter); - - try - { - SerializeHandler(writer, rootValue!); - writer.Flush(); - } - finally - { - // Record the serialization size in both successful and failed operations, - // since we want to immediately opt out of the fast path if it exceeds the threshold. - OnRootLevelAsyncSerializationCompleted(writer.BytesCommitted + writer.BytesPending); - - Utf8JsonWriterCache.ReturnWriter(writer); - } - - await bufferWriter.WriteToStreamAsync(utf8Json, cancellationToken).ConfigureAwait(false); - } - else if ( -#if NETCOREAPP - !typeof(T).IsValueType && -#endif - Converter.CanBePolymorphic && - rootValue is not null && - Options.TryGetPolymorphicTypeInfoForRootType(rootValue, out JsonTypeInfo? derivedTypeInfo)) - { - Debug.Assert(typeof(T) == typeof(object)); - await derivedTypeInfo.SerializeAsObjectAsync(utf8Json, rootValue, cancellationToken).ConfigureAwait(false); - } - else - { - bool isFinalBlock; - WriteStack state = default; - state.Initialize(this, - rootValueBoxed, - supportContinuation: true, - supportAsync: true); - - state.CancellationToken = cancellationToken; - - using PooledByteBufferWriter bufferWriter = new PooledByteBufferWriter(Options.DefaultBufferSize); - using Utf8JsonWriter writer = new Utf8JsonWriter(bufferWriter, Options.GetWriterOptions()); - - try - { - do - { - state.FlushThreshold = (int)(bufferWriter.Capacity * JsonSerializer.FlushThreshold); - - try - { - isFinalBlock = EffectiveConverter.WriteCore(writer, rootValue, Options, ref state); - writer.Flush(); - - if (state.SuppressFlush) - { - Debug.Assert(!isFinalBlock); - Debug.Assert(state.PendingTask is not null); - state.SuppressFlush = false; - } - else - { - await bufferWriter.WriteToStreamAsync(utf8Json, cancellationToken).ConfigureAwait(false); - bufferWriter.Clear(); - } - } - finally - { - // Await any pending resumable converter tasks (currently these can only be IAsyncEnumerator.MoveNextAsync() tasks). - // Note that pending tasks are always awaited, even if an exception has been thrown or the cancellation token has fired. - if (state.PendingTask is not null) - { - // Exceptions should only be propagated by the resuming converter -#if NET8_0_OR_GREATER - await state.PendingTask.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing); -#else - try - { - await state.PendingTask.ConfigureAwait(false); - } - catch { } -#endif - } - - // Dispose any pending async disposables (currently these can only be completed IAsyncEnumerators). - if (state.CompletedAsyncDisposables?.Count > 0) - { - await state.DisposeCompletedAsyncDisposables().ConfigureAwait(false); - } - } - - } while (!isFinalBlock); - } - catch - { - // On exception, walk the WriteStack for any orphaned disposables and try to dispose them. - await state.DisposePendingDisposablesOnExceptionAsync().ConfigureAwait(false); - throw; - } - - if (CanUseSerializeHandler) - { - // On successful serialization, record the serialization size - // to determine potential suitability of the type for - // fast-path serialization in streaming methods. - Debug.Assert(writer.BytesPending == 0); - OnRootLevelAsyncSerializationCompleted(writer.BytesCommitted); - } - } + return SerializeAsync(new AsyncSerializationStreamContext(utf8Json, Options), rootValue, cancellationToken, rootValueBoxed); } - internal async Task SerializeAsync( - PipeWriter utf8Json, + internal Task SerializeAsync(PipeWriter utf8Json, T? rootValue, CancellationToken cancellationToken, object? rootValueBoxed = null) + { + return SerializeAsync(new AsyncSerializationPipeContext(utf8Json), rootValue, cancellationToken, rootValueBoxed); + } + + // Root serialization method for async streaming serialization. + private async Task SerializeAsync( + TSerializationContext serializationContext, + T? rootValue, + CancellationToken cancellationToken, + object? rootValueBoxed = null) where TSerializationContext : struct, IAsyncSerializationBufferWriterContext { Debug.Assert(IsConfigured); Debug.Assert(rootValueBoxed is null || rootValueBoxed is T); @@ -201,11 +88,14 @@ internal async Task SerializeAsync( if (CanUseSerializeHandlerInStreaming) { // Short-circuit calls into SerializeHandler, if the `CanUseSerializeHandlerInStreaming` heuristic allows it. + Debug.Assert(SerializeHandler != null); Debug.Assert(CanUseSerializeHandler); Debug.Assert(Converter is JsonMetadataServicesConverter); - Utf8JsonWriter writer = Utf8JsonWriterCache.RentWriter(Options, utf8Json); + // Bufferwriter must be disposed after Utf8JsonWriter, use an unnamed variable to use cleaner using syntax. + using TSerializationContext _ = serializationContext; + Utf8JsonWriter writer = Utf8JsonWriterCache.RentWriter(Options, serializationContext.BufferWriter); try { @@ -221,7 +111,7 @@ internal async Task SerializeAsync( Utf8JsonWriterCache.ReturnWriter(writer); } - await utf8Json.FlushAsync(cancellationToken).ConfigureAwait(false); + await serializationContext.FlushAsync(cancellationToken).ConfigureAwait(false); } else if ( #if NETCOREAPP @@ -232,7 +122,7 @@ rootValue is not null && Options.TryGetPolymorphicTypeInfoForRootType(rootValue, out JsonTypeInfo? derivedTypeInfo)) { Debug.Assert(typeof(T) == typeof(object)); - await derivedTypeInfo.SerializeAsObjectAsync(utf8Json, rootValue, cancellationToken).ConfigureAwait(false); + await derivedTypeInfo.SerializeAsObjectAsync(serializationContext, rootValue, cancellationToken).ConfigureAwait(false); } else { @@ -245,13 +135,15 @@ rootValue is not null && state.CancellationToken = cancellationToken; - using Utf8JsonWriter writer = new Utf8JsonWriter(utf8Json, Options.GetWriterOptions()); + // Bufferwriter must be disposed after Utf8JsonWriter, use an unnamed variable to use cleaner using syntax. + using TSerializationContext _ = serializationContext; + using var writer = new Utf8JsonWriter(serializationContext.BufferWriter, Options.GetWriterOptions()); try { do { - state.FlushThreshold = (int)((4 * PipeOptions.Default.MinimumSegmentSize) * JsonSerializer.FlushThreshold); + state.FlushThreshold = serializationContext.FlushThreshold; try { @@ -266,11 +158,7 @@ rootValue is not null && } else { - FlushResult result = await utf8Json.FlushAsync(cancellationToken).ConfigureAwait(false); - if (result.IsCanceled || result.IsCompleted) - { - break; - } + await serializationContext.FlushAsync(cancellationToken).ConfigureAwait(false); } } finally @@ -371,8 +259,8 @@ rootValue is not null && supportContinuation: true, supportAsync: false); - using PooledByteBufferWriter bufferWriter = new PooledByteBufferWriter(Options.DefaultBufferSize); - using Utf8JsonWriter writer = new Utf8JsonWriter(bufferWriter, Options.GetWriterOptions()); + using var bufferWriter = new PooledByteBufferWriter(Options.DefaultBufferSize); + using var writer = new Utf8JsonWriter(bufferWriter, Options.GetWriterOptions()); do { @@ -401,10 +289,13 @@ rootValue is not null && internal sealed override void SerializeAsObject(Utf8JsonWriter writer, object? rootValue) => Serialize(writer, JsonSerializer.UnboxOnWrite(rootValue), rootValue); + internal sealed override Task SerializeAsObjectAsync(TSerializationContext serializationContext, object? rootValue, CancellationToken cancellationToken) + => SerializeAsync(serializationContext, JsonSerializer.UnboxOnWrite(rootValue), cancellationToken, rootValue); + internal sealed override Task SerializeAsObjectAsync(Stream utf8Json, object? rootValue, CancellationToken cancellationToken) => SerializeAsync(utf8Json, JsonSerializer.UnboxOnWrite(rootValue), cancellationToken, rootValue); - internal sealed override Task SerializeAsObjectAsync(System.IO.Pipelines.PipeWriter utf8Json, object? rootValue, CancellationToken cancellationToken) + internal sealed override Task SerializeAsObjectAsync(PipeWriter utf8Json, object? rootValue, CancellationToken cancellationToken) => SerializeAsync(utf8Json, JsonSerializer.UnboxOnWrite(rootValue), cancellationToken, rootValue); internal sealed override void SerializeAsObject(Stream utf8Json, object? rootValue) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.Serialization.cs b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.Serialization.cs index 5b05ff243a80a..07e28d1049224 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.Serialization.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.Serialization.cs @@ -885,5 +885,17 @@ public static void ThrowArgumentException_JsonPolymorphismOptionsAssociatedWithD { throw new ArgumentException(SR.JsonPolymorphismOptionsAssociatedWithDifferentJsonTypeInfo, paramName: parameterName); } + + [DoesNotReturn] + public static void ThrowOperationCanceledException_PipeWriteCanceled() + { + throw new OperationCanceledException(SR.PipeWriterCanceled); + } + + [DoesNotReturn] + public static void ThrowOperationCanceledException_PipeWriteCompleted() + { + throw new OperationCanceledException(SR.PipeWriterCompleted); + } } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Pipe.WriteTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Pipe.WriteTests.cs index ceffcb392d6b2..2914921699e26 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Pipe.WriteTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Pipe.WriteTests.cs @@ -39,13 +39,22 @@ public async Task VerifyTypeFail() public async Task CompletedPipeWithExceptionThrowsFromSerialize() { Pipe pipe = new Pipe(); - pipe.Reader.Complete(new Exception()); + pipe.Reader.Complete(new FormatException()); - await Assert.ThrowsAsync(() => JsonSerializer.SerializeAsync(pipe.Writer, 1)); + await Assert.ThrowsAsync(() => JsonSerializer.SerializeAsync(pipe.Writer, 1)); } [Fact] - public async Task CancelPendingFlushDuringBackpressureReturns() + public async Task CompletedPipeThrowsFromSerialize() + { + Pipe pipe = new Pipe(); + pipe.Reader.Complete(); + + await Assert.ThrowsAsync(() => JsonSerializer.SerializeAsync(pipe.Writer, 1)); + } + + [Fact] + public async Task CancelPendingFlushDuringBackpressureThrows() { Pipe pipe = new Pipe(new PipeOptions(pauseWriterThreshold: 10, resumeWriterThreshold: 5)); await pipe.Writer.WriteAsync("123456789"u8.ToArray()); @@ -54,9 +63,11 @@ public async Task CancelPendingFlushDuringBackpressureReturns() pipe.Writer.CancelPendingFlush(); - await serializeTask; + await Assert.ThrowsAsync(() => serializeTask); ReadResult result = await pipe.Reader.ReadAsync(); + + // Technically this check is not needed, but helps confirm behavior, that Pipe had written but was waiting for flush to continue. // result.Buffer: 123456789[0 Assert.Equal(11, result.Buffer.Length); pipe.Reader.AdvanceTo(result.Buffer.End); From 95cb59c311909a796464415a05cca07ba1a52d3b Mon Sep 17 00:00:00 2001 From: Brennan Date: Wed, 8 May 2024 18:02:49 -0700 Subject: [PATCH 11/14] minor --- .../Json/Serialization/AsyncSerializationBufferWriterContext.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncSerializationBufferWriterContext.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncSerializationBufferWriterContext.cs index 7cce1e44d6d78..ecbd9d4eb6e53 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncSerializationBufferWriterContext.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncSerializationBufferWriterContext.cs @@ -74,7 +74,7 @@ public async ValueTask FlushAsync(CancellationToken cancellationToken) ThrowHelper.ThrowOperationCanceledException_PipeWriteCanceled(); } - ThrowHelper.ThrowOperationCanceledException_PipeWriteCanceled(); + ThrowHelper.ThrowOperationCanceledException_PipeWriteCompleted(); } } From ad097e941fdf5587f60fa1f4891b931cf0f5beea Mon Sep 17 00:00:00 2001 From: Brennan Date: Thu, 9 May 2024 13:26:00 -0700 Subject: [PATCH 12/14] fb --- .../AsyncSerializationBufferWriterContext.cs | 2 ++ .../Metadata/JsonTypeInfoOfT.WriteHelpers.cs | 27 ++++++++++--------- .../Serialization/CollectionTests.cs | 5 ---- .../Serialization/InvalidTypeTests.cs | 5 ---- .../JsonCreationHandlingTests.cs | 5 ---- .../JsonSerializerWrapper.Reflection.cs | 19 +++---------- .../Serialization/PolymorphicTests.cs | 5 ---- .../Serialization/ReferenceHandlerTests.cs | 5 ---- .../Serialization/RequiredKeywordTests.cs | 5 ---- 9 files changed, 21 insertions(+), 57 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncSerializationBufferWriterContext.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncSerializationBufferWriterContext.cs index ecbd9d4eb6e53..f34bcaf612892 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncSerializationBufferWriterContext.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/AsyncSerializationBufferWriterContext.cs @@ -44,6 +44,7 @@ public async ValueTask FlushAsync(CancellationToken cancellationToken) public IBufferWriter BufferWriter => _bufferWriter; + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Dispose() { _bufferWriter.Dispose(); @@ -78,6 +79,7 @@ public async ValueTask FlushAsync(CancellationToken cancellationToken) } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Dispose() { } } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs index a92f7fdb60f3d..a93d22990ea3f 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs @@ -101,6 +101,7 @@ private async Task SerializeAsync( { SerializeHandler(writer, rootValue!); writer.Flush(); + await serializationContext.FlushAsync(cancellationToken).ConfigureAwait(false); } finally { @@ -109,9 +110,8 @@ private async Task SerializeAsync( OnRootLevelAsyncSerializationCompleted(writer.BytesCommitted + writer.BytesPending); Utf8JsonWriterCache.ReturnWriter(writer); + serializationContext.Dispose(); } - - await serializationContext.FlushAsync(cancellationToken).ConfigureAwait(false); } else if ( #if NET @@ -135,9 +135,7 @@ rootValue is not null && state.CancellationToken = cancellationToken; - // Bufferwriter must be disposed after Utf8JsonWriter, use an unnamed variable to use cleaner using syntax. - using TSerializationContext _ = serializationContext; - using var writer = new Utf8JsonWriter(serializationContext.BufferWriter, Options.GetWriterOptions()); + var writer = new Utf8JsonWriter(serializationContext.BufferWriter, Options.GetWriterOptions()); try { @@ -187,6 +185,15 @@ rootValue is not null && } } while (!isFinalBlock); + + if (CanUseSerializeHandler) + { + // On successful serialization, record the serialization size + // to determine potential suitability of the type for + // fast-path serialization in streaming methods. + Debug.Assert(writer.BytesPending == 0); + OnRootLevelAsyncSerializationCompleted(writer.BytesCommitted); + } } catch { @@ -194,14 +201,10 @@ rootValue is not null && await state.DisposePendingDisposablesOnExceptionAsync().ConfigureAwait(false); throw; } - - if (CanUseSerializeHandler) + finally { - // On successful serialization, record the serialization size - // to determine potential suitability of the type for - // fast-path serialization in streaming methods. - Debug.Assert(writer.BytesPending == 0); - OnRootLevelAsyncSerializationCompleted(writer.BytesCommitted); + writer.Dispose(); + serializationContext.Dispose(); } } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/CollectionTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/CollectionTests.cs index f8050941dadb2..73c4ed6f661c9 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/CollectionTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/CollectionTests.cs @@ -29,9 +29,4 @@ public sealed partial class CollectionTestsDynamic_Pipe : CollectionTests { public CollectionTestsDynamic_Pipe() : base(JsonSerializerWrapper.AsyncPipeSerializer) { } } - - public sealed partial class CollectionTestsDynamic_PipeWithSmallBuffer : CollectionTests - { - public CollectionTestsDynamic_PipeWithSmallBuffer() : base(JsonSerializerWrapper.AsyncPipeSerializerWithSmallBuffer) { } - } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/InvalidTypeTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/InvalidTypeTests.cs index 4b8be091ccb95..d4ea4d2f594c1 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/InvalidTypeTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/InvalidTypeTests.cs @@ -42,11 +42,6 @@ public class InvalidTypeTests_Pipe : InvalidTypeTests public InvalidTypeTests_Pipe() : base(JsonSerializerWrapper.AsyncPipeSerializer) { } } - public class InvalidTypeTests_PipeWithSmallBuffer : InvalidTypeTests - { - public InvalidTypeTests_PipeWithSmallBuffer() : base(JsonSerializerWrapper.AsyncPipeSerializerWithSmallBuffer) { } - } - public abstract class InvalidTypeTests { private JsonSerializerWrapper Serializer { get; } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonCreationHandlingTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonCreationHandlingTests.cs index b901cd079599c..573638913ae38 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonCreationHandlingTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonCreationHandlingTests.cs @@ -27,9 +27,4 @@ public sealed class JsonCreationHandlingTests_Pipe : JsonCreationHandlingTests { public JsonCreationHandlingTests_Pipe() : base(JsonSerializerWrapper.AsyncPipeSerializer) { } } - - public sealed class JsonCreationHandlingTests_PipeWithSmallBuffer : JsonCreationHandlingTests - { - public JsonCreationHandlingTests_PipeWithSmallBuffer() : base(JsonSerializerWrapper.AsyncPipeSerializerWithSmallBuffer) { } - } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerWrapper.Reflection.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerWrapper.Reflection.cs index af436e96b358a..a0a8167ec328c 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerWrapper.Reflection.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/JsonSerializerWrapper.Reflection.cs @@ -35,7 +35,6 @@ protected JsonSerializerWrapper() public static JsonSerializerWrapper ElementSerializer { get; } = new ElementSerializerWrapper(); public static JsonSerializerWrapper NodeSerializer { get; } = new NodeSerializerWrapper(); public static JsonSerializerWrapper AsyncPipeSerializer { get; } = new AsyncPipelinesSerializerWrapper(); - public static JsonSerializerWrapper AsyncPipeSerializerWithSmallBuffer { get; } = new AsyncPipelinesSerializerWrapper(forceSmallBufferInOptions: true); private class SpanSerializerWrapper : JsonSerializerWrapper { @@ -889,26 +888,16 @@ private int ReadExactlyFromSource(byte[] buffer, int offset, int count) // TODO: Deserialize to use PipeReader overloads once implemented private class AsyncPipelinesSerializerWrapper : JsonSerializerWrapper { - private readonly bool _forceSmallBufferInOptions; - public override JsonSerializerOptions DefaultOptions => JsonSerializerOptions.Default; public override bool SupportsNullValueOnDeserialize => true; - private JsonSerializerOptions? ResolveOptionsInstance(JsonSerializerOptions? options) - => _forceSmallBufferInOptions ? JsonSerializerOptionsSmallBufferMapper.ResolveOptionsInstanceWithSmallBuffer(options) : options; - - public AsyncPipelinesSerializerWrapper(bool forceSmallBufferInOptions = false) - { - _forceSmallBufferInOptions = forceSmallBufferInOptions; - } - public override async Task DeserializeWrapper(string json, JsonSerializerOptions options = null) { - return await JsonSerializer.DeserializeAsync(new MemoryStream(Encoding.UTF8.GetBytes(json)), ResolveOptionsInstance(options)); + return await JsonSerializer.DeserializeAsync(new MemoryStream(Encoding.UTF8.GetBytes(json)), options); } public override async Task DeserializeWrapper(string json, Type type, JsonSerializerOptions options = null) { - return await JsonSerializer.DeserializeAsync(new MemoryStream(Encoding.UTF8.GetBytes(json)), type, ResolveOptionsInstance(options)); + return await JsonSerializer.DeserializeAsync(new MemoryStream(Encoding.UTF8.GetBytes(json)), type, options); } public override async Task DeserializeWrapper(string json, JsonTypeInfo jsonTypeInfo) @@ -929,7 +918,7 @@ public override async Task DeserializeWrapper(string json, Type type, Js public override async Task SerializeWrapper(object value, Type inputType, JsonSerializerOptions options = null) { Pipe pipe = new Pipe(); - await JsonSerializer.SerializeAsync(pipe.Writer, value, inputType, ResolveOptionsInstance(options)); + await JsonSerializer.SerializeAsync(pipe.Writer, value, inputType, options); ReadResult result = await pipe.Reader.ReadAsync(); string stringResult = Encoding.UTF8.GetString(result.Buffer.ToArray()); @@ -940,7 +929,7 @@ public override async Task SerializeWrapper(object value, Type inputType public override async Task SerializeWrapper(T value, JsonSerializerOptions options = null) { Pipe pipe = new Pipe(); - await JsonSerializer.SerializeAsync(pipe.Writer, value, ResolveOptionsInstance(options)); + await JsonSerializer.SerializeAsync(pipe.Writer, value, options); ReadResult result = await pipe.Reader.ReadAsync(); string stringResult = Encoding.UTF8.GetString(result.Buffer.ToArray()); diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/PolymorphicTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/PolymorphicTests.cs index 2ce3fdbc39aee..9e14737051a2e 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/PolymorphicTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/PolymorphicTests.cs @@ -60,11 +60,6 @@ public class PolymorphicTests_Pipe : PolymorphicTests public PolymorphicTests_Pipe() : base(JsonSerializerWrapper.AsyncPipeSerializer) { } } - public class PolymorphicTests_PipeWithSmallBuffer : PolymorphicTests - { - public PolymorphicTests_PipeWithSmallBuffer() : base(JsonSerializerWrapper.AsyncPipeSerializerWithSmallBuffer) { } - } - public abstract partial class PolymorphicTests : SerializerTests { public PolymorphicTests(JsonSerializerWrapper serializer) : base(serializer) diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ReferenceHandlerTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ReferenceHandlerTests.cs index 366d9c0d84040..d793d4cbd47e7 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ReferenceHandlerTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/ReferenceHandlerTests.cs @@ -34,9 +34,4 @@ public sealed class ReferenceHandlerTestsDynamic_IgnoreCycles_Pipe : ReferenceHa { public ReferenceHandlerTestsDynamic_IgnoreCycles_Pipe() : base(JsonSerializerWrapper.AsyncPipeSerializer) { } } - - public sealed class ReferenceHandlerTestsDynamic_IgnoreCycles_PipeWithSmallBuffer : ReferenceHandlerTests_IgnoreCycles - { - public ReferenceHandlerTestsDynamic_IgnoreCycles_PipeWithSmallBuffer() : base(JsonSerializerWrapper.AsyncPipeSerializerWithSmallBuffer) { } - } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/RequiredKeywordTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/RequiredKeywordTests.cs index 42dfbe331f860..969f5d17c6063 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/RequiredKeywordTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/RequiredKeywordTests.cs @@ -57,9 +57,4 @@ public class RequiredKeywordTests_Pipe : RequiredKeywordTests { public RequiredKeywordTests_Pipe() : base(JsonSerializerWrapper.AsyncPipeSerializer) { } } - - public class RequiredKeywordTests_PipeWithSmallBuffer : RequiredKeywordTests - { - public RequiredKeywordTests_PipeWithSmallBuffer() : base(JsonSerializerWrapper.AsyncPipeSerializerWithSmallBuffer) { } - } } From 2e0d69d67027eca960c0cf20afbe7ed1da7f3db8 Mon Sep 17 00:00:00 2001 From: Brennan Date: Thu, 9 May 2024 16:44:37 -0700 Subject: [PATCH 13/14] no async! --- .../Metadata/JsonTypeInfoOfT.WriteHelpers.cs | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs index a93d22990ea3f..a5c2bf8745592 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfoOfT.WriteHelpers.cs @@ -93,23 +93,28 @@ private async Task SerializeAsync( Debug.Assert(CanUseSerializeHandler); Debug.Assert(Converter is JsonMetadataServicesConverter); - // Bufferwriter must be disposed after Utf8JsonWriter, use an unnamed variable to use cleaner using syntax. - using TSerializationContext _ = serializationContext; Utf8JsonWriter writer = Utf8JsonWriterCache.RentWriter(Options, serializationContext.BufferWriter); try { - SerializeHandler(writer, rootValue!); - writer.Flush(); + try + { + SerializeHandler(writer, rootValue!); + writer.Flush(); + } + finally + { + // Record the serialization size in both successful and failed operations, + // since we want to immediately opt out of the fast path if it exceeds the threshold. + OnRootLevelAsyncSerializationCompleted(writer.BytesCommitted + writer.BytesPending); + + Utf8JsonWriterCache.ReturnWriter(writer); + } + await serializationContext.FlushAsync(cancellationToken).ConfigureAwait(false); } finally { - // Record the serialization size in both successful and failed operations, - // since we want to immediately opt out of the fast path if it exceeds the threshold. - OnRootLevelAsyncSerializationCompleted(writer.BytesCommitted + writer.BytesPending); - - Utf8JsonWriterCache.ReturnWriter(writer); serializationContext.Dispose(); } } From 6a6d91b513670710e18d5460fd9127037100ba6f Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Sat, 11 May 2024 19:29:22 +0300 Subject: [PATCH 14/14] Update src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Pipe.WriteTests.cs Co-authored-by: David Fowler --- .../System.Text.Json.Tests/Serialization/Pipe.WriteTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Pipe.WriteTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Pipe.WriteTests.cs index 2914921699e26..6bd7a765e3829 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Pipe.WriteTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Pipe.WriteTests.cs @@ -300,7 +300,6 @@ public async Task BuffersBehaveAsExpected() result = await pipe.Reader.ReadAsync(); Assert.Equal(2002, result.Buffer.Length); - //pipe.Writer.CancelPendingFlush(); } internal class TestPool : MemoryPool