From 47748d1ea16106859c170c03e8fd72f71c34c4a0 Mon Sep 17 00:00:00 2001 From: Pranav Senthilnathan Date: Sun, 12 Jan 2025 12:17:46 -0800 Subject: [PATCH 1/6] Improve validation perf --- .../src/System/Text/Json/BitStack.cs | 2 +- .../Utf8JsonWriter.WriteProperties.Helpers.cs | 28 +++-- .../Utf8JsonWriter.WriteValues.Helpers.cs | 62 +++++++--- .../System/Text/Json/Writer/Utf8JsonWriter.cs | 106 +++++++++++++----- .../Utf8JsonWriterTests.cs | 58 ++++++++++ 5 files changed, 201 insertions(+), 55 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/BitStack.cs b/src/libraries/System.Text.Json/src/System/Text/Json/BitStack.cs index 40d70a56a5c4bf..32cb2016e01a4a 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/BitStack.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/BitStack.cs @@ -26,7 +26,7 @@ internal struct BitStack private int _currentDepth; - public int CurrentDepth => _currentDepth; + public readonly int CurrentDepth => _currentDepth; [MethodImpl(MethodImplOptions.AggressiveInlining)] public void PushTrue() diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Helpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Helpers.cs index b3b85281b82cc9..495c6a538a8d18 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Helpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Helpers.cs @@ -3,6 +3,7 @@ using System.Buffers; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; @@ -36,13 +37,10 @@ private void ValidateWritingProperty() { if (!_options.SkipValidation) { - // Make sure a new property is not attempted within an unfinalized string. - ValidateNotWithinUnfinalizedString(); - - if (!_inObject || _tokenType == JsonTokenType.PropertyName) + if (_enclosingContainer != EnclosingContainerType.Object || _tokenType == JsonTokenType.PropertyName || _tokenType == StringSegmentSentinel) { Debug.Assert(_tokenType != JsonTokenType.StartObject); - ThrowHelper.ThrowInvalidOperationException(ExceptionResource.CannotWritePropertyWithinArray, currentDepth: default, maxDepth: _options.MaxDepth, token: default, _tokenType); + OnValidateWritingPropertyFailed(); } } } @@ -52,18 +50,28 @@ private void ValidateWritingProperty(byte token) { if (!_options.SkipValidation) { - // Make sure a new property is not attempted within an unfinalized string. - ValidateNotWithinUnfinalizedString(); - - if (!_inObject || _tokenType == JsonTokenType.PropertyName) + if (_enclosingContainer != EnclosingContainerType.Object || _tokenType == JsonTokenType.PropertyName || _tokenType == StringSegmentSentinel) { Debug.Assert(_tokenType != JsonTokenType.StartObject); - ThrowHelper.ThrowInvalidOperationException(ExceptionResource.CannotWritePropertyWithinArray, currentDepth: default, maxDepth: _options.MaxDepth, token: default, _tokenType); + OnValidateWritingPropertyFailed(); } UpdateBitStackOnStart(token); } } + [DoesNotReturn] + [MethodImpl(MethodImplOptions.NoInlining)] + private void OnValidateWritingPropertyFailed() + { + if (_enclosingContainer != EnclosingContainerType.Object || _tokenType == JsonTokenType.PropertyName) + { + ThrowInvalidOperationException(ExceptionResource.CannotWritePropertyWithinArray); + } + + Debug.Assert(_tokenType == StringSegmentSentinel); + ThrowInvalidOperationException(ExceptionResource.CannotWriteWithinString); + } + private void WritePropertyNameMinimized(ReadOnlySpan escapedPropertyName, byte token) { Debug.Assert(escapedPropertyName.Length < int.MaxValue - 5); diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs index a8440144d4cf88..4748d11a2687f2 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs @@ -12,6 +12,29 @@ namespace System.Text.Json { public sealed partial class Utf8JsonWriter { + /// + /// Assuming that the writer is currently in a valid state, this returns true if a JSON value is not allowed at the current position. + /// Note that every JsonTokenType is less than 16 (0b0001_0000) except string segment (which is 0b0010_0000), so for these tokens only the + /// low nibble needs to be checked. There are 3 cases to consider: + /// + /// + /// The writer is in an array ( = 0b0001_0000): The only invalid previous token is a string segment. + /// ^ 0b0001_0000 is 0, so the entire expression is > 0b0001_0000, which is true iff the previous token is a string segment. + /// + /// + /// The writer is at the root level ( = 0). The only valid previous token is none. ^ 0b0001_0000 = 0b0001_0000, + /// so the entire expression is 0b0001_0000 ^ > 0b0001_0000. For string segment this is true, and for all other tokens we just need to check the low + /// nibble. 0000 ^ wxyz = 0 iff wxyz = 0000, which is JsonTokenType.None. For every other token, the inequality is true. + /// + /// + /// The writer is in an object ( = 0b0000_0101). The only valid previous token is a property. ^ 0b0001_0000 = 0b0001_0101, + /// so the entire expression is 0b0001_0101 ^ > 0b0001_0000. For string segment this inequality is true. For every other token, we just need + /// to check the low nibble. 0101 ^ wxyz = 0 iff wxyz = 0101, which is JsonTokenType.PropertyName. For every other token, the inequality is true. + /// + /// + /// + private bool CannotWriteValue => (0b0001_0000 ^ (byte)_enclosingContainer ^ (byte)_tokenType) > 0b0001_0000; + private bool HasPartialCodePoint => PartialCodePointLength != 0; private void ClearPartialCodePoint() => PartialCodePointLength = 0; @@ -28,7 +51,7 @@ private void ValidateNotWithinUnfinalizedString() { if (_tokenType == StringSegmentSentinel) { - ThrowHelper.ThrowInvalidOperationException(ExceptionResource.CannotWriteWithinString, currentDepth: default, maxDepth: _options.MaxDepth, token: default, _tokenType); + ThrowInvalidOperationException(ExceptionResource.CannotWriteWithinString); } Debug.Assert(PreviousSegmentEncoding == SegmentEncoding.None); @@ -36,29 +59,38 @@ private void ValidateNotWithinUnfinalizedString() } private void ValidateWritingValue() + { + if (CannotWriteValue) + { + OnValidateWritingValueFailed(); + } + } + + [DoesNotReturn] + [MethodImpl(MethodImplOptions.NoInlining)] + private void OnValidateWritingValueFailed() { Debug.Assert(!_options.SkipValidation); - // Make sure a new value is not attempted within an unfinalized string. - ValidateNotWithinUnfinalizedString(); + if (_tokenType == StringSegmentSentinel) + { + ThrowInvalidOperationException(ExceptionResource.CannotWriteWithinString); + } + + Debug.Assert(PreviousSegmentEncoding == SegmentEncoding.None); + Debug.Assert(!HasPartialCodePoint); - if (_inObject) + if (_enclosingContainer == EnclosingContainerType.Object) { - if (_tokenType != JsonTokenType.PropertyName) - { - Debug.Assert(_tokenType != JsonTokenType.None && _tokenType != JsonTokenType.StartArray); - ThrowHelper.ThrowInvalidOperationException(ExceptionResource.CannotWriteValueWithinObject, currentDepth: default, maxDepth: _options.MaxDepth, token: default, _tokenType); - } + Debug.Assert(_tokenType != JsonTokenType.PropertyName); + Debug.Assert(_tokenType != JsonTokenType.None && _tokenType != JsonTokenType.StartArray); + ThrowInvalidOperationException(ExceptionResource.CannotWriteValueWithinObject); } else { Debug.Assert(_tokenType != JsonTokenType.PropertyName); - - // It is more likely for CurrentDepth to not equal 0 when writing valid JSON, so check that first to rely on short-circuiting and return quickly. - if (CurrentDepth == 0 && _tokenType != JsonTokenType.None) - { - ThrowHelper.ThrowInvalidOperationException(ExceptionResource.CannotWriteValueAfterPrimitiveOrClose, currentDepth: default, maxDepth: _options.MaxDepth, token: default, _tokenType); - } + Debug.Assert(CurrentDepth == 0 && _tokenType != JsonTokenType.None); + ThrowInvalidOperationException(ExceptionResource.CannotWriteValueAfterPrimitiveOrClose); } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs index da9dc4b6bac503..29dc7dd0e00cd6 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs @@ -9,6 +9,7 @@ using System.Threading; using System.Threading.Tasks; using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; namespace System.Text.Json { @@ -36,7 +37,7 @@ public sealed partial class Utf8JsonWriter : IDisposable, IAsyncDisposable private const int InitialGrowthSize = 256; // A special value for JsonTokenType that lets the writer keep track of string segments. - private const JsonTokenType StringSegmentSentinel = (JsonTokenType)255; + private const JsonTokenType StringSegmentSentinel = (JsonTokenType)0b0010_0000; // Masks and flags for the length and encoding of the partial code point private const byte PartialCodePointLengthMask = 0b000_000_11; @@ -51,7 +52,7 @@ public sealed partial class Utf8JsonWriter : IDisposable, IAsyncDisposable private Memory _memory; - private bool _inObject; + private EnclosingContainerType _enclosingContainer; private bool _commentAfterNoneOrPropertyName; private JsonTokenType _tokenType; private BitStack _bitStack; @@ -383,7 +384,7 @@ private void ResetHelper() BytesCommitted = default; _memory = default; - _inObject = default; + _enclosingContainer = default; _tokenType = default; _commentAfterNoneOrPropertyName = default; _currentDepth = default; @@ -601,7 +602,7 @@ public void WriteStartObject() private void WriteStart(byte token) { if (CurrentDepth >= _options.MaxDepth) - ThrowHelper.ThrowInvalidOperationException(ExceptionResource.DepthTooLarge, _currentDepth, _options.MaxDepth, token: default, tokenType: default); + ThrowInvalidOperationException_DepthTooLarge(); if (_options.IndentedOrNotSkipValidation) { @@ -654,28 +655,38 @@ private void WriteStartSlow(byte token) } private void ValidateStart() + { + if (CannotWriteValue) + { + OnValidateStartFailed(); + } + } + + [DoesNotReturn] + [MethodImpl(MethodImplOptions.NoInlining)] + private void OnValidateStartFailed() { // Make sure a new object or array is not attempted within an unfinalized string. - ValidateNotWithinUnfinalizedString(); + if (_tokenType == StringSegmentSentinel) + { + ThrowInvalidOperationException(ExceptionResource.CannotWriteWithinString); + } + + Debug.Assert(PreviousSegmentEncoding == SegmentEncoding.None); + Debug.Assert(!HasPartialCodePoint); - if (_inObject) + if (_enclosingContainer == EnclosingContainerType.Object) { - if (_tokenType != JsonTokenType.PropertyName) - { - Debug.Assert(_tokenType != JsonTokenType.None && _tokenType != JsonTokenType.StartArray); - ThrowHelper.ThrowInvalidOperationException(ExceptionResource.CannotStartObjectArrayWithoutProperty, currentDepth: default, maxDepth: _options.MaxDepth, token: default, _tokenType); - } + Debug.Assert(_tokenType != JsonTokenType.PropertyName); + Debug.Assert(_tokenType != JsonTokenType.None && _tokenType != JsonTokenType.StartArray); + ThrowInvalidOperationException(ExceptionResource.CannotStartObjectArrayWithoutProperty); } else { Debug.Assert(_tokenType != JsonTokenType.PropertyName); Debug.Assert(_tokenType != JsonTokenType.StartObject); - - // It is more likely for CurrentDepth to not equal 0 when writing valid JSON, so check that first to rely on short-circuiting and return quickly. - if (CurrentDepth == 0 && _tokenType != JsonTokenType.None) - { - ThrowHelper.ThrowInvalidOperationException(ExceptionResource.CannotStartObjectArrayAfterPrimitiveOrClose, currentDepth: default, maxDepth: _options.MaxDepth, token: default, _tokenType); - } + Debug.Assert(CurrentDepth == 0 && _tokenType != JsonTokenType.None); + ThrowInvalidOperationException(ExceptionResource.CannotStartObjectArrayAfterPrimitiveOrClose); } } @@ -1081,33 +1092,33 @@ private void WriteEndSlow(byte token) } } + // Performance degrades significantly in some scenarios when inlining is allowed. + [MethodImpl(MethodImplOptions.NoInlining)] private void ValidateEnd(byte token) { - // Make sure an object is not ended within an unfinalized string. - ValidateNotWithinUnfinalizedString(); - - if (_bitStack.CurrentDepth <= 0 || _tokenType == JsonTokenType.PropertyName) - ThrowHelper.ThrowInvalidOperationException(ExceptionResource.MismatchedObjectArray, currentDepth: default, maxDepth: _options.MaxDepth, token, _tokenType); + if (_bitStack.CurrentDepth <= 0 || _tokenType == JsonTokenType.PropertyName || _tokenType == StringSegmentSentinel) + ThrowInvalidOperationException_MismatchedObjectArray(token); if (token == JsonConstants.CloseBracket) { - if (_inObject) + if (_enclosingContainer != EnclosingContainerType.Array) { Debug.Assert(_tokenType != JsonTokenType.None); - ThrowHelper.ThrowInvalidOperationException(ExceptionResource.MismatchedObjectArray, currentDepth: default, maxDepth: _options.MaxDepth, token, _tokenType); + ThrowInvalidOperationException_MismatchedObjectArray(token); } } else { Debug.Assert(token == JsonConstants.CloseBrace); - if (!_inObject) + if (_enclosingContainer != EnclosingContainerType.Object) { - ThrowHelper.ThrowInvalidOperationException(ExceptionResource.MismatchedObjectArray, currentDepth: default, maxDepth: _options.MaxDepth, token, _tokenType); + ThrowInvalidOperationException_MismatchedObjectArray(token); } } - _inObject = _bitStack.Pop(); + EnclosingContainerType container = _bitStack.Pop() ? EnclosingContainerType.Object : EnclosingContainerType.Array; + _enclosingContainer = _bitStack.CurrentDepth == 0 ? EnclosingContainerType.None : container; } private void WriteEndIndented(byte token) @@ -1173,13 +1184,13 @@ private void UpdateBitStackOnStart(byte token) if (token == JsonConstants.OpenBracket) { _bitStack.PushFalse(); - _inObject = false; + _enclosingContainer = EnclosingContainerType.Array; } else { Debug.Assert(token == JsonConstants.OpenBrace); _bitStack.PushTrue(); - _inObject = true; + _enclosingContainer = EnclosingContainerType.Object; } } @@ -1255,7 +1266,44 @@ private void SetFlagToAddListSeparatorBeforeNextItem() _currentDepth |= 1 << 31; } + [MethodImpl(MethodImplOptions.NoInlining)] + [DoesNotReturn] + private void ThrowInvalidOperationException(ExceptionResource resource) + => ThrowHelper.ThrowInvalidOperationException(resource, currentDepth: default, maxDepth: _options.MaxDepth, token: default, _tokenType); + + [MethodImpl(MethodImplOptions.NoInlining)] + [DoesNotReturn] + private void ThrowInvalidOperationException_MismatchedObjectArray(byte token) + => ThrowHelper.ThrowInvalidOperationException(ExceptionResource.MismatchedObjectArray, currentDepth: default, maxDepth: _options.MaxDepth, token, _tokenType); + + [MethodImpl(MethodImplOptions.NoInlining)] + [DoesNotReturn] + private void ThrowInvalidOperationException_DepthTooLarge() + => ThrowHelper.ThrowInvalidOperationException(ExceptionResource.DepthTooLarge, _currentDepth, _options.MaxDepth, token: default, tokenType: default); + [DebuggerBrowsable(DebuggerBrowsableState.Never)] private string DebuggerDisplay => $"BytesCommitted = {BytesCommitted} BytesPending = {BytesPending} CurrentDepth = {CurrentDepth}"; + + /// + /// The type of container that is enclosing the current position. The underlying values have been chosen + /// to allow validation to be done using bitwise operations and must be kept in sync with JsonTokenType. + /// + internal enum EnclosingContainerType : byte + { + /// + /// Root + /// + None = 0b0000_0000, + + /// + /// JSON object. Note that this is the same value as JsonTokenType.PropertyName. See for more details. + /// + Object = 0b0000_0101, + + /// + /// JSON array + /// + Array = 0b0001_0000, + } } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.cs index 382349214beb1b..04a3f4c36e607f 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.cs @@ -2879,6 +2879,64 @@ public void InvalidJsonStringValueSegment(string _, Action write Assert.Throws(() => write(jsonUtf8)); } } + + using (var jsonUtf8 = new Utf8JsonWriter(output, options)) + { + jsonUtf8.WriteStartArray(); + jsonUtf8.WriteStringValueSegment("foo"u8, isFinalSegment: false); + if (options.SkipValidation) + { + write(jsonUtf8); + } + else + { + Assert.Throws(() => write(jsonUtf8)); + } + } + + using (var jsonUtf8 = new Utf8JsonWriter(output, options)) + { + jsonUtf8.WriteStartArray(); + jsonUtf8.WriteStringValueSegment("foo".ToCharArray(), isFinalSegment: false); + if (options.SkipValidation) + { + write(jsonUtf8); + } + else + { + Assert.Throws(() => write(jsonUtf8)); + } + } + + using (var jsonUtf8 = new Utf8JsonWriter(output, options)) + { + jsonUtf8.WriteStartObject(); + jsonUtf8.WritePropertyName("prop"); + jsonUtf8.WriteStringValueSegment("foo"u8, isFinalSegment: false); + if (options.SkipValidation) + { + write(jsonUtf8); + } + else + { + Assert.Throws(() => write(jsonUtf8)); + } + } + + using (var jsonUtf8 = new Utf8JsonWriter(output, options)) + { + jsonUtf8.WriteStartObject(); + jsonUtf8.WritePropertyName("prop"); + jsonUtf8.WriteStringValueSegment("foo".ToCharArray(), isFinalSegment: false); + if (options.SkipValidation) + { + write(jsonUtf8); + } + else + { + Assert.Throws(() => write(jsonUtf8)); + } + } } [Theory] From f80bd84cfb4b4da95fc6b3572062244c210aff04 Mon Sep 17 00:00:00 2001 From: Pranav Senthilnathan Date: Sun, 12 Jan 2025 22:31:20 -0800 Subject: [PATCH 2/6] Improve JSON validation perf --- .../src/System/Text/Json/BitStack.cs | 2 +- .../Utf8JsonWriter.WriteProperties.Helpers.cs | 28 +++-- .../Utf8JsonWriter.WriteValues.Helpers.cs | 62 +++++++--- .../System/Text/Json/Writer/Utf8JsonWriter.cs | 107 +++++++++++++----- .../Utf8JsonWriterTests.cs | 58 ++++++++++ 5 files changed, 202 insertions(+), 55 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/BitStack.cs b/src/libraries/System.Text.Json/src/System/Text/Json/BitStack.cs index 40d70a56a5c4bf..32cb2016e01a4a 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/BitStack.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/BitStack.cs @@ -26,7 +26,7 @@ internal struct BitStack private int _currentDepth; - public int CurrentDepth => _currentDepth; + public readonly int CurrentDepth => _currentDepth; [MethodImpl(MethodImplOptions.AggressiveInlining)] public void PushTrue() diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Helpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Helpers.cs index b3b85281b82cc9..495c6a538a8d18 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Helpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Helpers.cs @@ -3,6 +3,7 @@ using System.Buffers; using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; @@ -36,13 +37,10 @@ private void ValidateWritingProperty() { if (!_options.SkipValidation) { - // Make sure a new property is not attempted within an unfinalized string. - ValidateNotWithinUnfinalizedString(); - - if (!_inObject || _tokenType == JsonTokenType.PropertyName) + if (_enclosingContainer != EnclosingContainerType.Object || _tokenType == JsonTokenType.PropertyName || _tokenType == StringSegmentSentinel) { Debug.Assert(_tokenType != JsonTokenType.StartObject); - ThrowHelper.ThrowInvalidOperationException(ExceptionResource.CannotWritePropertyWithinArray, currentDepth: default, maxDepth: _options.MaxDepth, token: default, _tokenType); + OnValidateWritingPropertyFailed(); } } } @@ -52,18 +50,28 @@ private void ValidateWritingProperty(byte token) { if (!_options.SkipValidation) { - // Make sure a new property is not attempted within an unfinalized string. - ValidateNotWithinUnfinalizedString(); - - if (!_inObject || _tokenType == JsonTokenType.PropertyName) + if (_enclosingContainer != EnclosingContainerType.Object || _tokenType == JsonTokenType.PropertyName || _tokenType == StringSegmentSentinel) { Debug.Assert(_tokenType != JsonTokenType.StartObject); - ThrowHelper.ThrowInvalidOperationException(ExceptionResource.CannotWritePropertyWithinArray, currentDepth: default, maxDepth: _options.MaxDepth, token: default, _tokenType); + OnValidateWritingPropertyFailed(); } UpdateBitStackOnStart(token); } } + [DoesNotReturn] + [MethodImpl(MethodImplOptions.NoInlining)] + private void OnValidateWritingPropertyFailed() + { + if (_enclosingContainer != EnclosingContainerType.Object || _tokenType == JsonTokenType.PropertyName) + { + ThrowInvalidOperationException(ExceptionResource.CannotWritePropertyWithinArray); + } + + Debug.Assert(_tokenType == StringSegmentSentinel); + ThrowInvalidOperationException(ExceptionResource.CannotWriteWithinString); + } + private void WritePropertyNameMinimized(ReadOnlySpan escapedPropertyName, byte token) { Debug.Assert(escapedPropertyName.Length < int.MaxValue - 5); diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs index a8440144d4cf88..4748d11a2687f2 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs @@ -12,6 +12,29 @@ namespace System.Text.Json { public sealed partial class Utf8JsonWriter { + /// + /// Assuming that the writer is currently in a valid state, this returns true if a JSON value is not allowed at the current position. + /// Note that every JsonTokenType is less than 16 (0b0001_0000) except string segment (which is 0b0010_0000), so for these tokens only the + /// low nibble needs to be checked. There are 3 cases to consider: + /// + /// + /// The writer is in an array ( = 0b0001_0000): The only invalid previous token is a string segment. + /// ^ 0b0001_0000 is 0, so the entire expression is > 0b0001_0000, which is true iff the previous token is a string segment. + /// + /// + /// The writer is at the root level ( = 0). The only valid previous token is none. ^ 0b0001_0000 = 0b0001_0000, + /// so the entire expression is 0b0001_0000 ^ > 0b0001_0000. For string segment this is true, and for all other tokens we just need to check the low + /// nibble. 0000 ^ wxyz = 0 iff wxyz = 0000, which is JsonTokenType.None. For every other token, the inequality is true. + /// + /// + /// The writer is in an object ( = 0b0000_0101). The only valid previous token is a property. ^ 0b0001_0000 = 0b0001_0101, + /// so the entire expression is 0b0001_0101 ^ > 0b0001_0000. For string segment this inequality is true. For every other token, we just need + /// to check the low nibble. 0101 ^ wxyz = 0 iff wxyz = 0101, which is JsonTokenType.PropertyName. For every other token, the inequality is true. + /// + /// + /// + private bool CannotWriteValue => (0b0001_0000 ^ (byte)_enclosingContainer ^ (byte)_tokenType) > 0b0001_0000; + private bool HasPartialCodePoint => PartialCodePointLength != 0; private void ClearPartialCodePoint() => PartialCodePointLength = 0; @@ -28,7 +51,7 @@ private void ValidateNotWithinUnfinalizedString() { if (_tokenType == StringSegmentSentinel) { - ThrowHelper.ThrowInvalidOperationException(ExceptionResource.CannotWriteWithinString, currentDepth: default, maxDepth: _options.MaxDepth, token: default, _tokenType); + ThrowInvalidOperationException(ExceptionResource.CannotWriteWithinString); } Debug.Assert(PreviousSegmentEncoding == SegmentEncoding.None); @@ -36,29 +59,38 @@ private void ValidateNotWithinUnfinalizedString() } private void ValidateWritingValue() + { + if (CannotWriteValue) + { + OnValidateWritingValueFailed(); + } + } + + [DoesNotReturn] + [MethodImpl(MethodImplOptions.NoInlining)] + private void OnValidateWritingValueFailed() { Debug.Assert(!_options.SkipValidation); - // Make sure a new value is not attempted within an unfinalized string. - ValidateNotWithinUnfinalizedString(); + if (_tokenType == StringSegmentSentinel) + { + ThrowInvalidOperationException(ExceptionResource.CannotWriteWithinString); + } + + Debug.Assert(PreviousSegmentEncoding == SegmentEncoding.None); + Debug.Assert(!HasPartialCodePoint); - if (_inObject) + if (_enclosingContainer == EnclosingContainerType.Object) { - if (_tokenType != JsonTokenType.PropertyName) - { - Debug.Assert(_tokenType != JsonTokenType.None && _tokenType != JsonTokenType.StartArray); - ThrowHelper.ThrowInvalidOperationException(ExceptionResource.CannotWriteValueWithinObject, currentDepth: default, maxDepth: _options.MaxDepth, token: default, _tokenType); - } + Debug.Assert(_tokenType != JsonTokenType.PropertyName); + Debug.Assert(_tokenType != JsonTokenType.None && _tokenType != JsonTokenType.StartArray); + ThrowInvalidOperationException(ExceptionResource.CannotWriteValueWithinObject); } else { Debug.Assert(_tokenType != JsonTokenType.PropertyName); - - // It is more likely for CurrentDepth to not equal 0 when writing valid JSON, so check that first to rely on short-circuiting and return quickly. - if (CurrentDepth == 0 && _tokenType != JsonTokenType.None) - { - ThrowHelper.ThrowInvalidOperationException(ExceptionResource.CannotWriteValueAfterPrimitiveOrClose, currentDepth: default, maxDepth: _options.MaxDepth, token: default, _tokenType); - } + Debug.Assert(CurrentDepth == 0 && _tokenType != JsonTokenType.None); + ThrowInvalidOperationException(ExceptionResource.CannotWriteValueAfterPrimitiveOrClose); } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs index da9dc4b6bac503..d2cbdb9b1cc184 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs @@ -9,6 +9,7 @@ using System.Threading; using System.Threading.Tasks; using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; namespace System.Text.Json { @@ -36,7 +37,7 @@ public sealed partial class Utf8JsonWriter : IDisposable, IAsyncDisposable private const int InitialGrowthSize = 256; // A special value for JsonTokenType that lets the writer keep track of string segments. - private const JsonTokenType StringSegmentSentinel = (JsonTokenType)255; + private const JsonTokenType StringSegmentSentinel = (JsonTokenType)0b0010_0000; // Masks and flags for the length and encoding of the partial code point private const byte PartialCodePointLengthMask = 0b000_000_11; @@ -51,7 +52,7 @@ public sealed partial class Utf8JsonWriter : IDisposable, IAsyncDisposable private Memory _memory; - private bool _inObject; + private EnclosingContainerType _enclosingContainer; private bool _commentAfterNoneOrPropertyName; private JsonTokenType _tokenType; private BitStack _bitStack; @@ -383,7 +384,7 @@ private void ResetHelper() BytesCommitted = default; _memory = default; - _inObject = default; + _enclosingContainer = default; _tokenType = default; _commentAfterNoneOrPropertyName = default; _currentDepth = default; @@ -601,7 +602,7 @@ public void WriteStartObject() private void WriteStart(byte token) { if (CurrentDepth >= _options.MaxDepth) - ThrowHelper.ThrowInvalidOperationException(ExceptionResource.DepthTooLarge, _currentDepth, _options.MaxDepth, token: default, tokenType: default); + ThrowInvalidOperationException_DepthTooLarge(); if (_options.IndentedOrNotSkipValidation) { @@ -654,28 +655,39 @@ private void WriteStartSlow(byte token) } private void ValidateStart() + { + // Note that Start[Array|Object] indicates the start of a value, so the same check can be used. + if (CannotWriteValue) + { + OnValidateStartFailed(); + } + } + + [DoesNotReturn] + [MethodImpl(MethodImplOptions.NoInlining)] + private void OnValidateStartFailed() { // Make sure a new object or array is not attempted within an unfinalized string. - ValidateNotWithinUnfinalizedString(); + if (_tokenType == StringSegmentSentinel) + { + ThrowInvalidOperationException(ExceptionResource.CannotWriteWithinString); + } + + Debug.Assert(PreviousSegmentEncoding == SegmentEncoding.None); + Debug.Assert(!HasPartialCodePoint); - if (_inObject) + if (_enclosingContainer == EnclosingContainerType.Object) { - if (_tokenType != JsonTokenType.PropertyName) - { - Debug.Assert(_tokenType != JsonTokenType.None && _tokenType != JsonTokenType.StartArray); - ThrowHelper.ThrowInvalidOperationException(ExceptionResource.CannotStartObjectArrayWithoutProperty, currentDepth: default, maxDepth: _options.MaxDepth, token: default, _tokenType); - } + Debug.Assert(_tokenType != JsonTokenType.PropertyName); + Debug.Assert(_tokenType != JsonTokenType.None && _tokenType != JsonTokenType.StartArray); + ThrowInvalidOperationException(ExceptionResource.CannotStartObjectArrayWithoutProperty); } else { Debug.Assert(_tokenType != JsonTokenType.PropertyName); Debug.Assert(_tokenType != JsonTokenType.StartObject); - - // It is more likely for CurrentDepth to not equal 0 when writing valid JSON, so check that first to rely on short-circuiting and return quickly. - if (CurrentDepth == 0 && _tokenType != JsonTokenType.None) - { - ThrowHelper.ThrowInvalidOperationException(ExceptionResource.CannotStartObjectArrayAfterPrimitiveOrClose, currentDepth: default, maxDepth: _options.MaxDepth, token: default, _tokenType); - } + Debug.Assert(CurrentDepth == 0 && _tokenType != JsonTokenType.None); + ThrowInvalidOperationException(ExceptionResource.CannotStartObjectArrayAfterPrimitiveOrClose); } } @@ -1081,33 +1093,33 @@ private void WriteEndSlow(byte token) } } + // Performance degrades significantly in some scenarios when inlining is allowed. + [MethodImpl(MethodImplOptions.NoInlining)] private void ValidateEnd(byte token) { - // Make sure an object is not ended within an unfinalized string. - ValidateNotWithinUnfinalizedString(); - - if (_bitStack.CurrentDepth <= 0 || _tokenType == JsonTokenType.PropertyName) - ThrowHelper.ThrowInvalidOperationException(ExceptionResource.MismatchedObjectArray, currentDepth: default, maxDepth: _options.MaxDepth, token, _tokenType); + if (_bitStack.CurrentDepth <= 0 || _tokenType == JsonTokenType.PropertyName || _tokenType == StringSegmentSentinel) + ThrowInvalidOperationException_MismatchedObjectArray(token); if (token == JsonConstants.CloseBracket) { - if (_inObject) + if (_enclosingContainer != EnclosingContainerType.Array) { Debug.Assert(_tokenType != JsonTokenType.None); - ThrowHelper.ThrowInvalidOperationException(ExceptionResource.MismatchedObjectArray, currentDepth: default, maxDepth: _options.MaxDepth, token, _tokenType); + ThrowInvalidOperationException_MismatchedObjectArray(token); } } else { Debug.Assert(token == JsonConstants.CloseBrace); - if (!_inObject) + if (_enclosingContainer != EnclosingContainerType.Object) { - ThrowHelper.ThrowInvalidOperationException(ExceptionResource.MismatchedObjectArray, currentDepth: default, maxDepth: _options.MaxDepth, token, _tokenType); + ThrowInvalidOperationException_MismatchedObjectArray(token); } } - _inObject = _bitStack.Pop(); + EnclosingContainerType container = _bitStack.Pop() ? EnclosingContainerType.Object : EnclosingContainerType.Array; + _enclosingContainer = _bitStack.CurrentDepth == 0 ? EnclosingContainerType.None : container; } private void WriteEndIndented(byte token) @@ -1173,13 +1185,13 @@ private void UpdateBitStackOnStart(byte token) if (token == JsonConstants.OpenBracket) { _bitStack.PushFalse(); - _inObject = false; + _enclosingContainer = EnclosingContainerType.Array; } else { Debug.Assert(token == JsonConstants.OpenBrace); _bitStack.PushTrue(); - _inObject = true; + _enclosingContainer = EnclosingContainerType.Object; } } @@ -1255,7 +1267,44 @@ private void SetFlagToAddListSeparatorBeforeNextItem() _currentDepth |= 1 << 31; } + [MethodImpl(MethodImplOptions.NoInlining)] + [DoesNotReturn] + private void ThrowInvalidOperationException(ExceptionResource resource) + => ThrowHelper.ThrowInvalidOperationException(resource, currentDepth: default, maxDepth: _options.MaxDepth, token: default, _tokenType); + + [MethodImpl(MethodImplOptions.NoInlining)] + [DoesNotReturn] + private void ThrowInvalidOperationException_MismatchedObjectArray(byte token) + => ThrowHelper.ThrowInvalidOperationException(ExceptionResource.MismatchedObjectArray, currentDepth: default, maxDepth: _options.MaxDepth, token, _tokenType); + + [MethodImpl(MethodImplOptions.NoInlining)] + [DoesNotReturn] + private void ThrowInvalidOperationException_DepthTooLarge() + => ThrowHelper.ThrowInvalidOperationException(ExceptionResource.DepthTooLarge, _currentDepth, _options.MaxDepth, token: default, tokenType: default); + [DebuggerBrowsable(DebuggerBrowsableState.Never)] private string DebuggerDisplay => $"BytesCommitted = {BytesCommitted} BytesPending = {BytesPending} CurrentDepth = {CurrentDepth}"; + + /// + /// The type of container that is enclosing the current position. The underlying values have been chosen + /// to allow validation to be done using bitwise operations and must be kept in sync with JsonTokenType. + /// + private enum EnclosingContainerType : byte + { + /// + /// Root + /// + None = 0b0000_0000, + + /// + /// JSON object. Note that this is the same value as JsonTokenType.PropertyName. See for more details. + /// + Object = 0b0000_0101, + + /// + /// JSON array + /// + Array = 0b0001_0000, + } } } diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.cs index 382349214beb1b..04a3f4c36e607f 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Utf8JsonWriterTests.cs @@ -2879,6 +2879,64 @@ public void InvalidJsonStringValueSegment(string _, Action write Assert.Throws(() => write(jsonUtf8)); } } + + using (var jsonUtf8 = new Utf8JsonWriter(output, options)) + { + jsonUtf8.WriteStartArray(); + jsonUtf8.WriteStringValueSegment("foo"u8, isFinalSegment: false); + if (options.SkipValidation) + { + write(jsonUtf8); + } + else + { + Assert.Throws(() => write(jsonUtf8)); + } + } + + using (var jsonUtf8 = new Utf8JsonWriter(output, options)) + { + jsonUtf8.WriteStartArray(); + jsonUtf8.WriteStringValueSegment("foo".ToCharArray(), isFinalSegment: false); + if (options.SkipValidation) + { + write(jsonUtf8); + } + else + { + Assert.Throws(() => write(jsonUtf8)); + } + } + + using (var jsonUtf8 = new Utf8JsonWriter(output, options)) + { + jsonUtf8.WriteStartObject(); + jsonUtf8.WritePropertyName("prop"); + jsonUtf8.WriteStringValueSegment("foo"u8, isFinalSegment: false); + if (options.SkipValidation) + { + write(jsonUtf8); + } + else + { + Assert.Throws(() => write(jsonUtf8)); + } + } + + using (var jsonUtf8 = new Utf8JsonWriter(output, options)) + { + jsonUtf8.WriteStartObject(); + jsonUtf8.WritePropertyName("prop"); + jsonUtf8.WriteStringValueSegment("foo".ToCharArray(), isFinalSegment: false); + if (options.SkipValidation) + { + write(jsonUtf8); + } + else + { + Assert.Throws(() => write(jsonUtf8)); + } + } } [Theory] From c6cff9b1679d02ef0dff52dd542fd85047186e6a Mon Sep 17 00:00:00 2001 From: Pranav Senthilnathan Date: Mon, 13 Jan 2025 11:32:18 -0800 Subject: [PATCH 3/6] change string segment from json token to enclosing container --- .../src/System/Text/Json/BitStack.cs | 6 ++-- .../Utf8JsonWriter.WriteProperties.Helpers.cs | 12 ++++---- .../Utf8JsonWriter.WriteValues.Helpers.cs | 29 +++++++++---------- ...tf8JsonWriter.WriteValues.StringSegment.cs | 28 ++++++++++-------- .../System/Text/Json/Writer/Utf8JsonWriter.cs | 17 ++++++----- 5 files changed, 49 insertions(+), 43 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/BitStack.cs b/src/libraries/System.Text.Json/src/System/Text/Json/BitStack.cs index 32cb2016e01a4a..bd3af90ce6e0b8 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/BitStack.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/BitStack.cs @@ -110,13 +110,13 @@ public bool Pop() } else { - inObject = PopFromArray(); + inObject = PeekInArray(); } return inObject; } [MethodImpl(MethodImplOptions.NoInlining)] - private bool PopFromArray() + private readonly bool PeekInArray() { int index = _currentDepth - AllocationFreeMaxDepth - 1; Debug.Assert(_array != null); @@ -129,6 +129,8 @@ private bool PopFromArray() return (_array[elementIndex] & (1 << extraBits)) != 0; } + public readonly bool Peek() => _currentDepth <= AllocationFreeMaxDepth ? (_allocationFreeContainer & 1) != 0 : PeekInArray(); + private void DoubleArray(int minSize) { Debug.Assert(_array != null); diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Helpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Helpers.cs index 495c6a538a8d18..825f9c0b6a9506 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Helpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Helpers.cs @@ -37,7 +37,7 @@ private void ValidateWritingProperty() { if (!_options.SkipValidation) { - if (_enclosingContainer != EnclosingContainerType.Object || _tokenType == JsonTokenType.PropertyName || _tokenType == StringSegmentSentinel) + if (_enclosingContainer != EnclosingContainerType.Object || _tokenType == JsonTokenType.PropertyName) { Debug.Assert(_tokenType != JsonTokenType.StartObject); OnValidateWritingPropertyFailed(); @@ -50,7 +50,7 @@ private void ValidateWritingProperty(byte token) { if (!_options.SkipValidation) { - if (_enclosingContainer != EnclosingContainerType.Object || _tokenType == JsonTokenType.PropertyName || _tokenType == StringSegmentSentinel) + if (_enclosingContainer != EnclosingContainerType.Object || _tokenType == JsonTokenType.PropertyName) { Debug.Assert(_tokenType != JsonTokenType.StartObject); OnValidateWritingPropertyFailed(); @@ -63,13 +63,13 @@ private void ValidateWritingProperty(byte token) [MethodImpl(MethodImplOptions.NoInlining)] private void OnValidateWritingPropertyFailed() { - if (_enclosingContainer != EnclosingContainerType.Object || _tokenType == JsonTokenType.PropertyName) + if (_enclosingContainer == EnclosingContainerType.PartialValue) { - ThrowInvalidOperationException(ExceptionResource.CannotWritePropertyWithinArray); + ThrowInvalidOperationException(ExceptionResource.CannotWriteWithinString); } - Debug.Assert(_tokenType == StringSegmentSentinel); - ThrowInvalidOperationException(ExceptionResource.CannotWriteWithinString); + Debug.Assert(_enclosingContainer != EnclosingContainerType.Object || _tokenType == JsonTokenType.PropertyName); + ThrowInvalidOperationException(ExceptionResource.CannotWritePropertyWithinArray); } private void WritePropertyNameMinimized(ReadOnlySpan escapedPropertyName, byte token) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs index 4748d11a2687f2..39e96ff8dfe523 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs @@ -13,27 +13,26 @@ namespace System.Text.Json public sealed partial class Utf8JsonWriter { /// - /// Assuming that the writer is currently in a valid state, this returns true if a JSON value is not allowed at the current position. - /// Note that every JsonTokenType is less than 16 (0b0001_0000) except string segment (which is 0b0010_0000), so for these tokens only the - /// low nibble needs to be checked. There are 3 cases to consider: + /// Assuming that the writer is currently in a valid state, this returns true if a JSON value is allowed at the current position. /// /// - /// The writer is in an array ( = 0b0001_0000): The only invalid previous token is a string segment. - /// ^ 0b0001_0000 is 0, so the entire expression is > 0b0001_0000, which is true iff the previous token is a string segment. + /// If is an array then writing a value is always allowed. /// /// - /// The writer is at the root level ( = 0). The only valid previous token is none. ^ 0b0001_0000 = 0b0001_0000, - /// so the entire expression is 0b0001_0000 ^ > 0b0001_0000. For string segment this is true, and for all other tokens we just need to check the low - /// nibble. 0000 ^ wxyz = 0 iff wxyz = 0000, which is JsonTokenType.None. For every other token, the inequality is true. + /// If is an object then writing a value is allowed only if is a property name. + /// Because we designed == , we can just check for equality. /// /// - /// The writer is in an object ( = 0b0000_0101). The only valid previous token is a property. ^ 0b0001_0000 = 0b0001_0101, - /// so the entire expression is 0b0001_0101 ^ > 0b0001_0000. For string segment this inequality is true. For every other token, we just need - /// to check the low nibble. 0101 ^ wxyz = 0 iff wxyz = 0101, which is JsonTokenType.PropertyName. For every other token, the inequality is true. + /// If is none (the root level) then writing a value is allowed only if is None (only + /// one value may be written at the root). This case is identical to the previous case. + /// + /// + /// If is a partial value, then it will never be a valid by construction. /// /// + /// This method performs better without short circuiting (this often gets inlined so using simple branch free code seems to have some benefits). /// - private bool CannotWriteValue => (0b0001_0000 ^ (byte)_enclosingContainer ^ (byte)_tokenType) > 0b0001_0000; + private bool CanWriteValue => _enclosingContainer == EnclosingContainerType.Array | (byte)_enclosingContainer == (byte)_tokenType; private bool HasPartialCodePoint => PartialCodePointLength != 0; @@ -49,7 +48,7 @@ private void ValidateEncodingDidNotChange(SegmentEncoding currentSegmentEncoding private void ValidateNotWithinUnfinalizedString() { - if (_tokenType == StringSegmentSentinel) + if (_enclosingContainer == EnclosingContainerType.PartialValue) { ThrowInvalidOperationException(ExceptionResource.CannotWriteWithinString); } @@ -60,7 +59,7 @@ private void ValidateNotWithinUnfinalizedString() private void ValidateWritingValue() { - if (CannotWriteValue) + if (!CanWriteValue) { OnValidateWritingValueFailed(); } @@ -72,7 +71,7 @@ private void OnValidateWritingValueFailed() { Debug.Assert(!_options.SkipValidation); - if (_tokenType == StringSegmentSentinel) + if (_enclosingContainer == EnclosingContainerType.PartialValue) { ThrowInvalidOperationException(ExceptionResource.CannotWriteWithinString); } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.StringSegment.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.StringSegment.cs index 08b517cce9648d..64a901b21a8922 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.StringSegment.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.StringSegment.cs @@ -31,7 +31,11 @@ public void WriteStringValueSegment(ReadOnlySpan value, bool isFinalSegmen { JsonWriterHelper.ValidateValue(value); - if (_tokenType != Utf8JsonWriter.StringSegmentSentinel) + if (_enclosingContainer == EnclosingContainerType.PartialValue) + { + ValidateEncodingDidNotChange(SegmentEncoding.Utf16); + } + else { Debug.Assert(PreviousSegmentEncoding == SegmentEncoding.None); Debug.Assert(!HasPartialCodePoint); @@ -44,11 +48,7 @@ public void WriteStringValueSegment(ReadOnlySpan value, bool isFinalSegmen WriteStringSegmentPrologue(); PreviousSegmentEncoding = SegmentEncoding.Utf16; - _tokenType = Utf8JsonWriter.StringSegmentSentinel; - } - else - { - ValidateEncodingDidNotChange(SegmentEncoding.Utf16); + _enclosingContainer = EnclosingContainerType.PartialValue; } // The steps to write a string segment are to complete the previous partial code point @@ -68,6 +68,8 @@ public void WriteStringValueSegment(ReadOnlySpan value, bool isFinalSegmen SetFlagToAddListSeparatorBeforeNextItem(); PreviousSegmentEncoding = SegmentEncoding.None; + EnclosingContainerType container = _bitStack.Peek() ? EnclosingContainerType.Object : EnclosingContainerType.Array; + _enclosingContainer = _bitStack.CurrentDepth == 0 ? EnclosingContainerType.None : container; _tokenType = JsonTokenType.String; } } @@ -207,7 +209,11 @@ public void WriteStringValueSegment(ReadOnlySpan value, bool isFinalSegmen { JsonWriterHelper.ValidateValue(value); - if (_tokenType != Utf8JsonWriter.StringSegmentSentinel) + if (_enclosingContainer == EnclosingContainerType.PartialValue) + { + ValidateEncodingDidNotChange(SegmentEncoding.Utf8); + } + else { Debug.Assert(PreviousSegmentEncoding == SegmentEncoding.None); Debug.Assert(!HasPartialCodePoint); @@ -220,11 +226,7 @@ public void WriteStringValueSegment(ReadOnlySpan value, bool isFinalSegmen WriteStringSegmentPrologue(); PreviousSegmentEncoding = SegmentEncoding.Utf8; - _tokenType = Utf8JsonWriter.StringSegmentSentinel; - } - else - { - ValidateEncodingDidNotChange(SegmentEncoding.Utf8); + _enclosingContainer = EnclosingContainerType.PartialValue; } // The steps to write a string segment are to complete the previous partial code point @@ -244,6 +246,8 @@ public void WriteStringValueSegment(ReadOnlySpan value, bool isFinalSegmen SetFlagToAddListSeparatorBeforeNextItem(); PreviousSegmentEncoding = SegmentEncoding.None; + EnclosingContainerType container = _bitStack.Peek() ? EnclosingContainerType.Object : EnclosingContainerType.Array; + _enclosingContainer = _bitStack.CurrentDepth == 0 ? EnclosingContainerType.None : container; _tokenType = JsonTokenType.String; } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs index d2cbdb9b1cc184..26bd8e38e22750 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs @@ -36,9 +36,6 @@ public sealed partial class Utf8JsonWriter : IDisposable, IAsyncDisposable private const int DefaultGrowthSize = 4096; private const int InitialGrowthSize = 256; - // A special value for JsonTokenType that lets the writer keep track of string segments. - private const JsonTokenType StringSegmentSentinel = (JsonTokenType)0b0010_0000; - // Masks and flags for the length and encoding of the partial code point private const byte PartialCodePointLengthMask = 0b000_000_11; private const byte PartialCodePointEncodingMask = 0b000_111_00; @@ -657,7 +654,7 @@ private void WriteStartSlow(byte token) private void ValidateStart() { // Note that Start[Array|Object] indicates the start of a value, so the same check can be used. - if (CannotWriteValue) + if (!CanWriteValue) { OnValidateStartFailed(); } @@ -668,7 +665,7 @@ private void ValidateStart() private void OnValidateStartFailed() { // Make sure a new object or array is not attempted within an unfinalized string. - if (_tokenType == StringSegmentSentinel) + if (_enclosingContainer == EnclosingContainerType.PartialValue) { ThrowInvalidOperationException(ExceptionResource.CannotWriteWithinString); } @@ -1097,14 +1094,13 @@ private void WriteEndSlow(byte token) [MethodImpl(MethodImplOptions.NoInlining)] private void ValidateEnd(byte token) { - if (_bitStack.CurrentDepth <= 0 || _tokenType == JsonTokenType.PropertyName || _tokenType == StringSegmentSentinel) + if (_tokenType == JsonTokenType.PropertyName) ThrowInvalidOperationException_MismatchedObjectArray(token); if (token == JsonConstants.CloseBracket) { if (_enclosingContainer != EnclosingContainerType.Array) { - Debug.Assert(_tokenType != JsonTokenType.None); ThrowInvalidOperationException_MismatchedObjectArray(token); } } @@ -1297,7 +1293,7 @@ private enum EnclosingContainerType : byte None = 0b0000_0000, /// - /// JSON object. Note that this is the same value as JsonTokenType.PropertyName. See for more details. + /// JSON object. Note that this is the same value as JsonTokenType.PropertyName. See for more details. /// Object = 0b0000_0101, @@ -1305,6 +1301,11 @@ private enum EnclosingContainerType : byte /// JSON array /// Array = 0b0001_0000, + + /// + /// Partial value (currently only string value segment) + /// + PartialValue = 0b0010_0000, } } } From 36df713bc160c5052183057ff43fa9497c6e07d2 Mon Sep 17 00:00:00 2001 From: Pranav Senthilnathan Date: Tue, 14 Jan 2025 18:15:50 -0800 Subject: [PATCH 4/6] fix merge and remove PartialCodePointEncoding in favor of augmenting EnclosingContainerType --- .../src/System/Text/Json/ThrowHelper.cs | 19 +++- .../Utf8JsonWriter.WriteProperties.Helpers.cs | 2 +- .../Utf8JsonWriter.WriteValues.Helpers.cs | 64 ++++++++--- ...tf8JsonWriter.WriteValues.StringSegment.cs | 70 ++++-------- .../System/Text/Json/Writer/Utf8JsonWriter.cs | 105 +++++++----------- 5 files changed, 128 insertions(+), 132 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs index 2bc50fcfeb4d39..41048d8f0639cd 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Runtime.CompilerServices; +using static System.Text.Json.Utf8JsonWriter; namespace System.Text.Json { @@ -312,9 +313,23 @@ public static void ThrowInvalidOperationException_CannotSkipOnPartial() } [DoesNotReturn] - public static void ThrowInvalidOperationException_CannotMixEncodings(Utf8JsonWriter.SegmentEncoding previousEncoding, Utf8JsonWriter.SegmentEncoding currentEncoding) + [MethodImpl(MethodImplOptions.NoInlining)] + public static void ThrowInvalidOperationException_CannotMixEncodings(EnclosingContainerType previousEncoding, EnclosingContainerType currentEncoding) { - throw GetInvalidOperationException(SR.Format(SR.CannotMixEncodings, previousEncoding, currentEncoding)); + throw GetInvalidOperationException(SR.Format(SR.CannotMixEncodings, GetEncodingName(previousEncoding), GetEncodingName(currentEncoding))); + + static string GetEncodingName(EnclosingContainerType encoding) + { + switch (encoding) + { + case EnclosingContainerType.PartialUtf8String: return "UTF-8"; + case EnclosingContainerType.PartialUtf16String: return "UTF-16"; + case EnclosingContainerType.PartialBase64String: return "Base64"; + default: + Debug.Fail("Unknown encoding."); + return "Unknown"; + }; + } } private static InvalidOperationException GetInvalidOperationException(string message, JsonTokenType tokenType) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Helpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Helpers.cs index 825f9c0b6a9506..808255ae2a9687 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Helpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteProperties.Helpers.cs @@ -63,7 +63,7 @@ private void ValidateWritingProperty(byte token) [MethodImpl(MethodImplOptions.NoInlining)] private void OnValidateWritingPropertyFailed() { - if (_enclosingContainer == EnclosingContainerType.PartialValue) + if (IsWritingPartialString) { ThrowInvalidOperationException(ExceptionResource.CannotWriteWithinString); } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs index 372e5c0e9b4da7..dddd50353fe2cf 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs @@ -34,50 +34,68 @@ public sealed partial class Utf8JsonWriter /// private bool CanWriteValue => _enclosingContainer == EnclosingContainerType.Array | (byte)_enclosingContainer == (byte)_tokenType; - private bool HasPartialStringData => PartialStringDataLength != 0; + private bool HasPartialStringData => _partialStringDataLength != 0; - private void ClearPartialStringData() => PartialStringDataLength = 0; + private void ClearPartialStringData() => _partialStringDataLength = 0; - private void ValidateEncodingDidNotChange(SegmentEncoding currentSegmentEncoding) + private void ValidateWritingValue() { - if (PreviousSegmentEncoding != currentSegmentEncoding) + if (!CanWriteValue) { - ThrowHelper.ThrowInvalidOperationException_CannotMixEncodings(PreviousSegmentEncoding, currentSegmentEncoding); + OnValidateWritingValueFailed(); } } - private void ValidateNotWithinUnfinalizedString() + [DoesNotReturn] + [MethodImpl(MethodImplOptions.NoInlining)] + private void OnValidateWritingValueFailed() { - if (_enclosingContainer == EnclosingContainerType.PartialValue) + Debug.Assert(!_options.SkipValidation); + + if (IsWritingPartialString) { ThrowInvalidOperationException(ExceptionResource.CannotWriteWithinString); } - Debug.Assert(PreviousSegmentEncoding == SegmentEncoding.None); Debug.Assert(!HasPartialStringData); + + if (_enclosingContainer == EnclosingContainerType.Object) + { + Debug.Assert(_tokenType != JsonTokenType.PropertyName); + Debug.Assert(_tokenType != JsonTokenType.None && _tokenType != JsonTokenType.StartArray); + ThrowInvalidOperationException(ExceptionResource.CannotWriteValueWithinObject); + } + else + { + Debug.Assert(_tokenType != JsonTokenType.PropertyName); + Debug.Assert(CurrentDepth == 0 && _tokenType != JsonTokenType.None); + ThrowInvalidOperationException(ExceptionResource.CannotWriteValueAfterPrimitiveOrClose); + } } - private void ValidateWritingValue() + private void ValidateWritingSegment(EnclosingContainerType currentSegmentEncoding) { - if (!CanWriteValue) + // A string segment can be written if either: + // 1) The writer is currently in a partial string of the same type. In this case the new segment + // will continue the partial string. + // - or - + // 2) The writer can write a value at the current position, in which case a new string can be started. + if (_enclosingContainer != currentSegmentEncoding && !CanWriteValue) { - OnValidateWritingValueFailed(); + OnValidateWritingSegmentFailed(currentSegmentEncoding); } } [DoesNotReturn] [MethodImpl(MethodImplOptions.NoInlining)] - private void OnValidateWritingValueFailed() + private void OnValidateWritingSegmentFailed(EnclosingContainerType currentSegmentEncoding) { - Debug.Assert(!_options.SkipValidation); - - if (_enclosingContainer == EnclosingContainerType.PartialValue) + if (IsWritingPartialString) { - ThrowInvalidOperationException(ExceptionResource.CannotWriteWithinString); + ThrowHelper.ThrowInvalidOperationException_CannotMixEncodings(_enclosingContainer, currentSegmentEncoding); } - Debug.Assert(PreviousSegmentEncoding == SegmentEncoding.None); - Debug.Assert(!HasPartialCodePoint); + Debug.Assert(!HasPartialStringData); if (_enclosingContainer == EnclosingContainerType.Object) { @@ -93,6 +111,16 @@ private void OnValidateWritingValueFailed() } } + private void ValidateNotWithinUnfinalizedString() + { + if (IsWritingPartialString) + { + ThrowInvalidOperationException(ExceptionResource.CannotWriteWithinString); + } + + Debug.Assert(!HasPartialStringData); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private void Base64EncodeAndWrite(ReadOnlySpan bytes, Span output) { diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.StringSegment.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.StringSegment.cs index 98fc48b22c11c6..76aecce169a1b7 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.StringSegment.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.StringSegment.cs @@ -28,24 +28,15 @@ public void WriteStringValueSegment(ReadOnlySpan value, bool isFinalSegmen { JsonWriterHelper.ValidateValue(value); - if (_enclosingContainer == EnclosingContainerType.PartialValue) + if (!_options.SkipValidation) { - ValidateEncodingDidNotChange(SegmentEncoding.Utf16); + ValidateWritingSegment(EnclosingContainerType.PartialUtf16String); } - else - { - Debug.Assert(PreviousSegmentEncoding == SegmentEncoding.None); - Debug.Assert(!HasPartialStringData); - - if (!_options.SkipValidation) - { - ValidateWritingValue(); - } + if (_enclosingContainer != EnclosingContainerType.PartialUtf16String) + { WriteStringSegmentPrologue(); - - PreviousSegmentEncoding = SegmentEncoding.Utf16; - _enclosingContainer = EnclosingContainerType.PartialValue; + _enclosingContainer = EnclosingContainerType.PartialUtf16String; } // The steps to write a string segment are to complete the previous partial code point @@ -64,7 +55,6 @@ public void WriteStringValueSegment(ReadOnlySpan value, bool isFinalSegmen WriteStringSegmentEpilogue(); SetFlagToAddListSeparatorBeforeNextItem(); - PreviousSegmentEncoding = SegmentEncoding.None; EnclosingContainerType container = _bitStack.Peek() ? EnclosingContainerType.Object : EnclosingContainerType.Array; _enclosingContainer = _bitStack.CurrentDepth == 0 ? EnclosingContainerType.None : container; _tokenType = JsonTokenType.String; @@ -74,7 +64,7 @@ public void WriteStringValueSegment(ReadOnlySpan value, bool isFinalSegmen private void WriteStringSegmentWithLeftover(scoped ReadOnlySpan value, bool isFinalSegment) { Debug.Assert(HasPartialStringData); - Debug.Assert(PreviousSegmentEncoding == SegmentEncoding.Utf16); + Debug.Assert(_enclosingContainer == EnclosingContainerType.PartialUtf16String); scoped ReadOnlySpan partialStringDataBuffer = PartialUtf16StringData; @@ -206,24 +196,15 @@ public void WriteStringValueSegment(ReadOnlySpan value, bool isFinalSegmen { JsonWriterHelper.ValidateValue(value); - if (_enclosingContainer == EnclosingContainerType.PartialValue) + if (!_options.SkipValidation) { - ValidateEncodingDidNotChange(SegmentEncoding.Utf8); + ValidateWritingSegment(EnclosingContainerType.PartialUtf8String); } - else - { - Debug.Assert(PreviousSegmentEncoding == SegmentEncoding.None); - Debug.Assert(!HasPartialStringData); - - if (!_options.SkipValidation) - { - ValidateWritingValue(); - } + if (_enclosingContainer != EnclosingContainerType.PartialUtf8String) + { WriteStringSegmentPrologue(); - - PreviousSegmentEncoding = SegmentEncoding.Utf8; - _enclosingContainer = EnclosingContainerType.PartialValue; + _enclosingContainer = EnclosingContainerType.PartialUtf8String; } // The steps to write a string segment are to complete the previous partial code point @@ -242,7 +223,6 @@ public void WriteStringValueSegment(ReadOnlySpan value, bool isFinalSegmen WriteStringSegmentEpilogue(); SetFlagToAddListSeparatorBeforeNextItem(); - PreviousSegmentEncoding = SegmentEncoding.None; EnclosingContainerType container = _bitStack.Peek() ? EnclosingContainerType.Object : EnclosingContainerType.Array; _enclosingContainer = _bitStack.CurrentDepth == 0 ? EnclosingContainerType.None : container; _tokenType = JsonTokenType.String; @@ -252,7 +232,7 @@ public void WriteStringValueSegment(ReadOnlySpan value, bool isFinalSegmen private void WriteStringSegmentWithLeftover(scoped ReadOnlySpan utf8Value, bool isFinalSegment) { Debug.Assert(HasPartialStringData); - Debug.Assert(PreviousSegmentEncoding == SegmentEncoding.Utf8); + Debug.Assert(_enclosingContainer == EnclosingContainerType.PartialUtf8String); scoped ReadOnlySpan partialStringDataBuffer = PartialUtf8StringData; @@ -383,24 +363,15 @@ public void WriteBase64StringSegment(ReadOnlySpan value, bool isFinalSegme ThrowHelper.ThrowArgumentException_ValueTooLarge(value.Length); } - if (_tokenType != Utf8JsonWriter.StringSegmentSentinel) + if (!_options.SkipValidation) { - Debug.Assert(PreviousSegmentEncoding == SegmentEncoding.None); - Debug.Assert(!HasPartialStringData); - - if (!_options.SkipValidation) - { - ValidateWritingValue(); - } - - WriteStringSegmentPrologue(); - - PreviousSegmentEncoding = SegmentEncoding.Base64; - _tokenType = Utf8JsonWriter.StringSegmentSentinel; + ValidateWritingSegment(EnclosingContainerType.PartialBase64String); } - else + + if (_enclosingContainer != EnclosingContainerType.PartialBase64String) { - ValidateEncodingDidNotChange(SegmentEncoding.Base64); + WriteStringSegmentPrologue(); + _enclosingContainer = EnclosingContainerType.PartialBase64String; } // The steps to write a string segment are to complete the previous partial string data @@ -419,7 +390,8 @@ public void WriteBase64StringSegment(ReadOnlySpan value, bool isFinalSegme WriteStringSegmentEpilogue(); SetFlagToAddListSeparatorBeforeNextItem(); - PreviousSegmentEncoding = SegmentEncoding.None; + EnclosingContainerType container = _bitStack.Peek() ? EnclosingContainerType.Object : EnclosingContainerType.Array; + _enclosingContainer = _bitStack.CurrentDepth == 0 ? EnclosingContainerType.None : container; _tokenType = JsonTokenType.String; } } @@ -427,7 +399,7 @@ public void WriteBase64StringSegment(ReadOnlySpan value, bool isFinalSegme private void WriteBase64StringSegmentWithLeftover(scoped ReadOnlySpan bytes, bool isFinalSegment) { Debug.Assert(HasPartialStringData); - Debug.Assert(PreviousSegmentEncoding == SegmentEncoding.Base64); + Debug.Assert(_enclosingContainer == EnclosingContainerType.PartialBase64String); scoped ReadOnlySpan partialStringDataBuffer = PartialBase64StringData; diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs index 0f09596a57eb8a..133f0ed94df289 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs @@ -35,14 +35,6 @@ public sealed partial class Utf8JsonWriter : IDisposable, IAsyncDisposable private const int DefaultGrowthSize = 4096; private const int InitialGrowthSize = 256; - // Masks and flags for the length and encoding of the partial string data. - private const byte PartialStringDataLengthMask = 0b000_000_11; - private const byte PartialStringDataEncodingMask = 0b000_111_00; - - private const byte PartialStringDataUtf8EncodingFlag = 0b000_001_00; - private const byte PartialStringDataUtf16EncodingFlag = 0b000_010_00; - private const byte PartialStringDataBase64EncodingFlag = 0b000_100_00; - private IBufferWriter? _output; private Stream? _stream; private ArrayBufferWriter? _arrayBufferWriter; @@ -73,11 +65,9 @@ private struct Inline3ByteArray #endif /// - /// Stores the length and encoding of the partial string data. Outside of segment writes, this value is 0. - /// Across segment writes, this value is always non-zero even if the length is 0, to indicate the encoding of the segment. - /// This allows detection of encoding changes across segment writes. + /// Length of the partial string data. /// - private byte _partialStringDataFlags; + private byte _partialStringDataLength; // The highest order bit of _currentDepth is used to discern whether we are writing the first item in a list or not. // if (_currentDepth >> 31) == 1, add a list separator before writing the item @@ -125,15 +115,6 @@ private struct Inline3ByteArray /// public int CurrentDepth => _currentDepth & JsonConstants.RemoveFlagsBitMask; - /// - /// Length of the partial string data. - /// - private byte PartialStringDataLength - { - get => (byte)(_partialStringDataFlags & PartialStringDataLengthMask); - set => _partialStringDataFlags = (byte)((_partialStringDataFlags & ~PartialStringDataLengthMask) | value); - } - /// /// The partial UTF-8 code point. /// @@ -141,12 +122,12 @@ private ReadOnlySpan PartialUtf8StringData { get { - Debug.Assert(PreviousSegmentEncoding == SegmentEncoding.Utf8); + Debug.Assert(_enclosingContainer == EnclosingContainerType.PartialUtf8String); ReadOnlySpan partialStringDataBytes = PartialStringDataRaw; Debug.Assert(partialStringDataBytes.Length == 3); - byte length = PartialStringDataLength; + byte length = _partialStringDataLength; Debug.Assert(length < 4); return partialStringDataBytes.Slice(0, length); @@ -159,7 +140,7 @@ private ReadOnlySpan PartialUtf8StringData Span partialStringDataBytes = PartialStringDataRaw; value.CopyTo(partialStringDataBytes); - PartialStringDataLength = (byte)value.Length; + _partialStringDataLength = (byte)value.Length; } } @@ -170,12 +151,12 @@ private ReadOnlySpan PartialUtf16StringData { get { - Debug.Assert(PreviousSegmentEncoding == SegmentEncoding.Utf16); + Debug.Assert(_enclosingContainer == EnclosingContainerType.PartialUtf16String); ReadOnlySpan partialStringDataBytes = PartialStringDataRaw; Debug.Assert(partialStringDataBytes.Length == 3); - byte length = PartialStringDataLength; + byte length = _partialStringDataLength; Debug.Assert(length is 2 or 0); return MemoryMarshal.Cast(partialStringDataBytes.Slice(0, length)); @@ -187,7 +168,7 @@ private ReadOnlySpan PartialUtf16StringData Span partialStringDataBytes = PartialStringDataRaw; value.CopyTo(MemoryMarshal.Cast(partialStringDataBytes)); - PartialStringDataLength = (byte)(2 * value.Length); + _partialStringDataLength = (byte)(2 * value.Length); } } @@ -198,12 +179,12 @@ private ReadOnlySpan PartialBase64StringData { get { - Debug.Assert(PreviousSegmentEncoding == SegmentEncoding.Base64); + Debug.Assert(_enclosingContainer == EnclosingContainerType.PartialBase64String); ReadOnlySpan partialStringDataBytes = PartialStringDataRaw; Debug.Assert(partialStringDataBytes.Length == 3); - byte length = PartialStringDataLength; + byte length = _partialStringDataLength; Debug.Assert(length < 3); return partialStringDataBytes.Slice(0, length); @@ -215,30 +196,10 @@ private ReadOnlySpan PartialBase64StringData Span partialStringDataBytes = PartialStringDataRaw; value.CopyTo(partialStringDataBytes); - PartialStringDataLength = (byte)value.Length; + _partialStringDataLength = (byte)value.Length; } } - /// - /// Encoding used for the previous string segment write. - /// - private SegmentEncoding PreviousSegmentEncoding - { - get => (SegmentEncoding)(_partialStringDataFlags & PartialStringDataEncodingMask); - set => _partialStringDataFlags = (byte)((_partialStringDataFlags & ~PartialStringDataEncodingMask) | (byte)value); - } - - /// - /// Convenience enumeration to track the encoding of the partial string data. This must be kept in sync with the PartialStringData*Encoding flags. - /// - internal enum SegmentEncoding : byte - { - None = 0, - Utf8 = PartialStringDataUtf8EncodingFlag, - Utf16 = PartialStringDataUtf16EncodingFlag, - Base64 = PartialStringDataBase64EncodingFlag, - } - private Utf8JsonWriter() { } @@ -418,7 +379,7 @@ private void ResetHelper() _bitStack = default; _partialStringData = default; - _partialStringDataFlags = default; + _partialStringDataLength = default; } private void CheckNotDisposed() @@ -694,13 +655,12 @@ private void ValidateStart() private void OnValidateStartFailed() { // Make sure a new object or array is not attempted within an unfinalized string. - if (_enclosingContainer == EnclosingContainerType.PartialValue) + if (IsWritingPartialString) { ThrowInvalidOperationException(ExceptionResource.CannotWriteWithinString); } - Debug.Assert(PreviousSegmentEncoding == SegmentEncoding.None); - Debug.Assert(!HasPartialCodePoint); + Debug.Assert(!HasPartialStringData); if (_enclosingContainer == EnclosingContainerType.Object) { @@ -1310,31 +1270,52 @@ private void ThrowInvalidOperationException_DepthTooLarge() [DebuggerBrowsable(DebuggerBrowsableState.Never)] private string DebuggerDisplay => $"BytesCommitted = {BytesCommitted} BytesPending = {BytesPending} CurrentDepth = {CurrentDepth}"; + /// + /// Indicates whether the writer is currently writing a partial string value. + /// + private bool IsWritingPartialString => _enclosingContainer >= EnclosingContainerType.PartialUtf8String; + /// /// The type of container that is enclosing the current position. The underlying values have been chosen - /// to allow validation to be done using bitwise operations and must be kept in sync with JsonTokenType. + /// to allow validation to be done using bitwise operations and must be kept in sync with . /// - private enum EnclosingContainerType : byte + internal enum EnclosingContainerType : byte { /// /// Root /// - None = 0b0000_0000, + None = 0b000_0_0000, /// - /// JSON object. Note that this is the same value as JsonTokenType.PropertyName. See for more details. + /// JSON object. Note that this is the same value as . See for more details. /// - Object = 0b0000_0101, + Object = 0b000_0_0101, /// /// JSON array /// - Array = 0b0001_0000, + Array = 0b000_1_0000, + + /// + /// Partial UTF-8 string. This is a container if viewed as an array of "utf-8 string segment"-typed values. This array can only be one level deep + /// so does not need to store its state. + /// relies on the value of the partial string members being the largest values of this enum. + /// + PartialUtf8String = 0b001_0_0000, + + /// + /// Partial UTF-16 string. This is a container if viewed as an array of "utf-16 string segment"-typed values. This array can only be one level deep + /// so does not need to store its state. + /// relies on the value of the partial string members being the largest values of this enum. + /// + PartialUtf16String = 0b010_0_0000, /// - /// Partial value (currently only string value segment) + /// Partial Base64 string. This is a container if viewed as an array of "base64 string segment"-typed values. This array can only be one level deep + /// so does not need to store its state. + /// relies on the value of the partial string members being the largest values of this enum. /// - PartialValue = 0b0010_0000, + PartialBase64String = 0b011_0_0000, } } } From adca8ae8ea5453c883d470a53357e0cf006b2f24 Mon Sep 17 00:00:00 2001 From: Pranav Senthilnathan Date: Thu, 16 Jan 2025 18:36:04 -0800 Subject: [PATCH 5/6] PR comments --- .../src/System/Text/Json/BitStack.cs | 38 ++++++++++++++++++- .../src/System/Text/Json/JsonTokenType.cs | 4 +- .../src/System/Text/Json/ThrowHelper.cs | 6 +-- .../Utf8JsonWriter.WriteValues.Helpers.cs | 15 +++++--- ...tf8JsonWriter.WriteValues.StringSegment.cs | 24 ++++++------ .../System/Text/Json/Writer/Utf8JsonWriter.cs | 28 +++++++------- 6 files changed, 77 insertions(+), 38 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/BitStack.cs b/src/libraries/System.Text.Json/src/System/Text/Json/BitStack.cs index bd3af90ce6e0b8..e96622e16952d6 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/BitStack.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/BitStack.cs @@ -14,6 +14,7 @@ internal struct BitStack private const int DefaultInitialArraySize = 2; + // The backing array for the stack used when the depth exceeds AllocationFreeMaxDepth. private int[]? _array; // This ulong container represents a tiny stack to track the state during nested transitions. @@ -26,8 +27,14 @@ internal struct BitStack private int _currentDepth; + /// + /// Gets the number of elements in the stack. + /// public readonly int CurrentDepth => _currentDepth; + /// + /// Pushes onto the stack. + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public void PushTrue() { @@ -42,6 +49,9 @@ public void PushTrue() _currentDepth++; } + /// + /// Pushes onto the stack. + /// [MethodImpl(MethodImplOptions.AggressiveInlining)] public void PushFalse() { @@ -56,7 +66,10 @@ public void PushFalse() _currentDepth++; } - // Allocate the bit array lazily only when it is absolutely necessary + /// + /// Pushes a bit onto the stack. Allocate the bit array lazily only when it is absolutely necessary. + /// + /// The bit to push onto the stack. [MethodImpl(MethodImplOptions.NoInlining)] private void PushToArray(bool value) { @@ -94,6 +107,10 @@ private void PushToArray(bool value) _array[elementIndex] = newValue; } + /// + /// Pops the bit at the top of the stack and returns its value. + /// + /// The bit that was popped. [MethodImpl(MethodImplOptions.AggressiveInlining)] public bool Pop() { @@ -110,11 +127,16 @@ public bool Pop() } else { + // Decrementing depth above effectively pops the last element in the array-backed case. inObject = PeekInArray(); } return inObject; } + /// + /// If the stack has a backing array allocated, this method will find the topmost bit in the array and return its value. + /// This should only be called if the depth is greater than AllocationFreeMaxDepth and an array has been allocated. + /// [MethodImpl(MethodImplOptions.NoInlining)] private readonly bool PeekInArray() { @@ -129,7 +151,13 @@ private readonly bool PeekInArray() return (_array[elementIndex] & (1 << extraBits)) != 0; } - public readonly bool Peek() => _currentDepth <= AllocationFreeMaxDepth ? (_allocationFreeContainer & 1) != 0 : PeekInArray(); + /// + /// Peeks at the bit at the top of the stack. + /// + /// The bit at the top of the stack. + public readonly bool Peek() + // If the stack is small enough, we can use the allocation-free container, otherwise check the allocated array. + => _currentDepth <= AllocationFreeMaxDepth ? (_allocationFreeContainer & 1) != 0 : PeekInArray(); private void DoubleArray(int minSize) { @@ -143,6 +171,9 @@ private void DoubleArray(int minSize) Array.Resize(ref _array, nextDouble); } + /// + /// Optimization to push as the first bit when the stack is empty. + /// public void SetFirstBit() { Debug.Assert(_currentDepth == 0, "Only call SetFirstBit when depth is 0"); @@ -150,6 +181,9 @@ public void SetFirstBit() _allocationFreeContainer = 1; } + /// + /// Optimization to push as the first bit when the stack is empty. + /// public void ResetFirstBit() { Debug.Assert(_currentDepth == 0, "Only call ResetFirstBit when depth is 0"); diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/JsonTokenType.cs b/src/libraries/System.Text.Json/src/System/Text/Json/JsonTokenType.cs index 049da2220a22f8..85164b36d32e77 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/JsonTokenType.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/JsonTokenType.cs @@ -12,8 +12,8 @@ namespace System.Text.Json /// public enum JsonTokenType : byte { - // Do not re-order. - // We rely on the ordering to quickly check things like IsTokenTypePrimitive + // Do not re-number. + // We rely on the underlying values to quickly check things like JsonReaderHelper.IsTokenTypePrimitive and Utf8JsonWriter.CanWriteValue /// /// Indicates that there is no value (as distinct from ). diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs index 41048d8f0639cd..e6a7790d2b757b 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/ThrowHelper.cs @@ -322,9 +322,9 @@ static string GetEncodingName(EnclosingContainerType encoding) { switch (encoding) { - case EnclosingContainerType.PartialUtf8String: return "UTF-8"; - case EnclosingContainerType.PartialUtf16String: return "UTF-16"; - case EnclosingContainerType.PartialBase64String: return "Base64"; + case EnclosingContainerType.Utf8StringSequence: return "UTF-8"; + case EnclosingContainerType.Utf16StringSequence: return "UTF-16"; + case EnclosingContainerType.Base64StringSequence: return "Base64"; default: Debug.Fail("Unknown encoding."); return "Unknown"; diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs index dddd50353fe2cf..1a450b476919c0 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.Helpers.cs @@ -13,21 +13,22 @@ namespace System.Text.Json public sealed partial class Utf8JsonWriter { /// - /// Assuming that the writer is currently in a valid state, this returns true if a JSON value is allowed at the current position. + /// Returns whether a JSON value can be written at the current position based on the current : /// /// - /// If is an array then writing a value is always allowed. + /// : Writing a value is always allowed. /// /// - /// If is an object then writing a value is allowed only if is a property name. + /// : Writing a value is allowed only if is a property name. /// Because we designed == , we can just check for equality. /// /// - /// If is none (the root level) then writing a value is allowed only if is None (only - /// one value may be written at the root). This case is identical to the previous case. + /// : Writing a value is allowed only if is None (only one value may be written at the root). + /// This case is identical to the previous case. /// /// - /// If is a partial value, then it will never be a valid by construction. + /// , , : + /// Writing a value is never valid and does not equal any by construction. /// /// /// This method performs better without short circuiting (this often gets inlined so using simple branch free code seems to have some benefits). @@ -75,6 +76,8 @@ private void OnValidateWritingValueFailed() private void ValidateWritingSegment(EnclosingContainerType currentSegmentEncoding) { + Debug.Assert(currentSegmentEncoding is EnclosingContainerType.Utf8StringSequence or EnclosingContainerType.Utf16StringSequence or EnclosingContainerType.Base64StringSequence); + // A string segment can be written if either: // 1) The writer is currently in a partial string of the same type. In this case the new segment // will continue the partial string. diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.StringSegment.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.StringSegment.cs index 76aecce169a1b7..1016a9f76ff25e 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.StringSegment.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.WriteValues.StringSegment.cs @@ -30,13 +30,13 @@ public void WriteStringValueSegment(ReadOnlySpan value, bool isFinalSegmen if (!_options.SkipValidation) { - ValidateWritingSegment(EnclosingContainerType.PartialUtf16String); + ValidateWritingSegment(EnclosingContainerType.Utf16StringSequence); } - if (_enclosingContainer != EnclosingContainerType.PartialUtf16String) + if (_enclosingContainer != EnclosingContainerType.Utf16StringSequence) { WriteStringSegmentPrologue(); - _enclosingContainer = EnclosingContainerType.PartialUtf16String; + _enclosingContainer = EnclosingContainerType.Utf16StringSequence; } // The steps to write a string segment are to complete the previous partial code point @@ -64,7 +64,7 @@ public void WriteStringValueSegment(ReadOnlySpan value, bool isFinalSegmen private void WriteStringSegmentWithLeftover(scoped ReadOnlySpan value, bool isFinalSegment) { Debug.Assert(HasPartialStringData); - Debug.Assert(_enclosingContainer == EnclosingContainerType.PartialUtf16String); + Debug.Assert(_enclosingContainer == EnclosingContainerType.Utf16StringSequence); scoped ReadOnlySpan partialStringDataBuffer = PartialUtf16StringData; @@ -198,13 +198,13 @@ public void WriteStringValueSegment(ReadOnlySpan value, bool isFinalSegmen if (!_options.SkipValidation) { - ValidateWritingSegment(EnclosingContainerType.PartialUtf8String); + ValidateWritingSegment(EnclosingContainerType.Utf8StringSequence); } - if (_enclosingContainer != EnclosingContainerType.PartialUtf8String) + if (_enclosingContainer != EnclosingContainerType.Utf8StringSequence) { WriteStringSegmentPrologue(); - _enclosingContainer = EnclosingContainerType.PartialUtf8String; + _enclosingContainer = EnclosingContainerType.Utf8StringSequence; } // The steps to write a string segment are to complete the previous partial code point @@ -232,7 +232,7 @@ public void WriteStringValueSegment(ReadOnlySpan value, bool isFinalSegmen private void WriteStringSegmentWithLeftover(scoped ReadOnlySpan utf8Value, bool isFinalSegment) { Debug.Assert(HasPartialStringData); - Debug.Assert(_enclosingContainer == EnclosingContainerType.PartialUtf8String); + Debug.Assert(_enclosingContainer == EnclosingContainerType.Utf8StringSequence); scoped ReadOnlySpan partialStringDataBuffer = PartialUtf8StringData; @@ -365,13 +365,13 @@ public void WriteBase64StringSegment(ReadOnlySpan value, bool isFinalSegme if (!_options.SkipValidation) { - ValidateWritingSegment(EnclosingContainerType.PartialBase64String); + ValidateWritingSegment(EnclosingContainerType.Base64StringSequence); } - if (_enclosingContainer != EnclosingContainerType.PartialBase64String) + if (_enclosingContainer != EnclosingContainerType.Base64StringSequence) { WriteStringSegmentPrologue(); - _enclosingContainer = EnclosingContainerType.PartialBase64String; + _enclosingContainer = EnclosingContainerType.Base64StringSequence; } // The steps to write a string segment are to complete the previous partial string data @@ -399,7 +399,7 @@ public void WriteBase64StringSegment(ReadOnlySpan value, bool isFinalSegme private void WriteBase64StringSegmentWithLeftover(scoped ReadOnlySpan bytes, bool isFinalSegment) { Debug.Assert(HasPartialStringData); - Debug.Assert(_enclosingContainer == EnclosingContainerType.PartialBase64String); + Debug.Assert(_enclosingContainer == EnclosingContainerType.Base64StringSequence); scoped ReadOnlySpan partialStringDataBuffer = PartialBase64StringData; diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs index 133f0ed94df289..44ccb6ffd1fa2e 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs @@ -122,7 +122,7 @@ private ReadOnlySpan PartialUtf8StringData { get { - Debug.Assert(_enclosingContainer == EnclosingContainerType.PartialUtf8String); + Debug.Assert(_enclosingContainer == EnclosingContainerType.Utf8StringSequence); ReadOnlySpan partialStringDataBytes = PartialStringDataRaw; Debug.Assert(partialStringDataBytes.Length == 3); @@ -151,7 +151,7 @@ private ReadOnlySpan PartialUtf16StringData { get { - Debug.Assert(_enclosingContainer == EnclosingContainerType.PartialUtf16String); + Debug.Assert(_enclosingContainer == EnclosingContainerType.Utf16StringSequence); ReadOnlySpan partialStringDataBytes = PartialStringDataRaw; Debug.Assert(partialStringDataBytes.Length == 3); @@ -179,7 +179,7 @@ private ReadOnlySpan PartialBase64StringData { get { - Debug.Assert(_enclosingContainer == EnclosingContainerType.PartialBase64String); + Debug.Assert(_enclosingContainer == EnclosingContainerType.Base64StringSequence); ReadOnlySpan partialStringDataBytes = PartialStringDataRaw; Debug.Assert(partialStringDataBytes.Length == 3); @@ -589,7 +589,9 @@ public void WriteStartObject() private void WriteStart(byte token) { if (CurrentDepth >= _options.MaxDepth) + { ThrowInvalidOperationException_DepthTooLarge(); + } if (_options.IndentedOrNotSkipValidation) { @@ -1273,26 +1275,26 @@ private void ThrowInvalidOperationException_DepthTooLarge() /// /// Indicates whether the writer is currently writing a partial string value. /// - private bool IsWritingPartialString => _enclosingContainer >= EnclosingContainerType.PartialUtf8String; + private bool IsWritingPartialString => _enclosingContainer >= EnclosingContainerType.Utf8StringSequence; /// /// The type of container that is enclosing the current position. The underlying values have been chosen - /// to allow validation to be done using bitwise operations and must be kept in sync with . + /// to allow to be done using bitwise operations and must be kept in sync with . /// internal enum EnclosingContainerType : byte { /// - /// Root + /// Root level. /// - None = 0b000_0_0000, + None = JsonTokenType.None, /// - /// JSON object. Note that this is the same value as . See for more details. + /// JSON object. /// - Object = 0b000_0_0101, + Object = JsonTokenType.PropertyName, /// - /// JSON array + /// JSON array. This is chosen large enough to not conflict with any value. /// Array = 0b000_1_0000, @@ -1301,21 +1303,21 @@ internal enum EnclosingContainerType : byte /// so does not need to store its state. /// relies on the value of the partial string members being the largest values of this enum. /// - PartialUtf8String = 0b001_0_0000, + Utf8StringSequence = 0b001_0_0000, /// /// Partial UTF-16 string. This is a container if viewed as an array of "utf-16 string segment"-typed values. This array can only be one level deep /// so does not need to store its state. /// relies on the value of the partial string members being the largest values of this enum. /// - PartialUtf16String = 0b010_0_0000, + Utf16StringSequence = 0b010_0_0000, /// /// Partial Base64 string. This is a container if viewed as an array of "base64 string segment"-typed values. This array can only be one level deep /// so does not need to store its state. /// relies on the value of the partial string members being the largest values of this enum. /// - PartialBase64String = 0b011_0_0000, + Base64StringSequence = 0b011_0_0000, } } } From 5901725bc44b9baddc9fd74c474f2df824de0444 Mon Sep 17 00:00:00 2001 From: Pranav Senthilnathan Date: Fri, 17 Jan 2025 10:20:43 -0800 Subject: [PATCH 6/6] clean up enum values/comments --- .../src/System/Text/Json/JsonTokenType.cs | 24 +++++++++---------- .../System/Text/Json/Writer/Utf8JsonWriter.cs | 16 +++++++------ 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/JsonTokenType.cs b/src/libraries/System.Text.Json/src/System/Text/Json/JsonTokenType.cs index 85164b36d32e77..be2d8fcfdcbf76 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/JsonTokenType.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/JsonTokenType.cs @@ -21,61 +21,61 @@ public enum JsonTokenType : byte /// /// This is the default token type if no data has been read by the . /// - None, + None = 0, /// /// Indicates that the token type is the start of a JSON object. /// - StartObject, + StartObject = 1, /// /// Indicates that the token type is the end of a JSON object. /// - EndObject, + EndObject = 2, /// /// Indicates that the token type is the start of a JSON array. /// - StartArray, + StartArray = 3, /// /// Indicates that the token type is the end of a JSON array. /// - EndArray, + EndArray = 4, /// /// Indicates that the token type is a JSON property name. /// - PropertyName, + PropertyName = 5, /// /// Indicates that the token type is the comment string. /// - Comment, + Comment = 6, /// /// Indicates that the token type is a JSON string. /// - String, + String = 7, /// /// Indicates that the token type is a JSON number. /// - Number, + Number = 8, /// /// Indicates that the token type is the JSON literal true. /// - True, + True = 9, /// /// Indicates that the token type is the JSON literal false. /// - False, + False = 10, /// /// Indicates that the token type is the JSON literal null. /// - Null, + Null = 11, } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs index 44ccb6ffd1fa2e..1fa6f661732d7e 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Writer/Utf8JsonWriter.cs @@ -1284,40 +1284,42 @@ private void ThrowInvalidOperationException_DepthTooLarge() internal enum EnclosingContainerType : byte { /// - /// Root level. + /// Root level. The choice of allows fast validation by equality comparison when writing values + /// since a value can be written at the root level only if there was no previous token. /// None = JsonTokenType.None, /// - /// JSON object. + /// JSON object. The choice of allows fast validation by equality comparison when writing values + /// since a value can be written inside a JSON object only if the previous token is a property name. /// Object = JsonTokenType.PropertyName, /// - /// JSON array. This is chosen large enough to not conflict with any value. + /// JSON array. Chosen so that its lower nibble is 0 to ensure it does not conflict with numeric values that currently are less than 16. /// - Array = 0b000_1_0000, + Array = 0x10, /// /// Partial UTF-8 string. This is a container if viewed as an array of "utf-8 string segment"-typed values. This array can only be one level deep /// so does not need to store its state. /// relies on the value of the partial string members being the largest values of this enum. /// - Utf8StringSequence = 0b001_0_0000, + Utf8StringSequence = 0x20, /// /// Partial UTF-16 string. This is a container if viewed as an array of "utf-16 string segment"-typed values. This array can only be one level deep /// so does not need to store its state. /// relies on the value of the partial string members being the largest values of this enum. /// - Utf16StringSequence = 0b010_0_0000, + Utf16StringSequence = 0x30, /// /// Partial Base64 string. This is a container if viewed as an array of "base64 string segment"-typed values. This array can only be one level deep /// so does not need to store its state. /// relies on the value of the partial string members being the largest values of this enum. /// - Base64StringSequence = 0b011_0_0000, + Base64StringSequence = 0x40, } } }