Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Query: Adds type-markers with count and length for large arrays #3852

Merged
8 commits merged into from
May 23, 2023
Merged

Query: Adds type-markers with count and length for large arrays #3852

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
5746764
initial commit
Maya-Painter May 15, 2023
ffb68d0
cleanup
Maya-Painter May 16, 2023
2f043f1
update test output
Maya-Painter May 16, 2023
58a303b
Merge branch 'master' into binaryencodingupdates
Maya-Painter May 16, 2023
9928082
cleanup
Maya-Painter May 16, 2023
75de8e7
typo
Maya-Painter May 16, 2023
608c96b
Pr comments
Maya-Painter May 22, 2023
740e517
Merge branch 'master' into binaryencodingupdates
Maya-Painter May 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 22 additions & 29 deletions Microsoft.Azure.Cosmos/src/Json/JsonWriter.JsonBinaryWriter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ abstract partial class JsonWriter : IJsonWriter
/// <returns>Blitted bytes.</returns>
public static PreblittedBinaryJsonScope CapturePreblittedBinaryJsonScope(Action<ITypedBinaryJsonWriter> scopeWriter)
{
JsonBinaryWriter jsonBinaryWriter = new JsonBinaryWriter(initialCapacity: 256, serializeCount: false, enableEncodedStrings: false);
JsonBinaryWriter jsonBinaryWriter = new JsonBinaryWriter(initialCapacity: 256, enableEncodedStrings: false);
Contract.Requires(!jsonBinaryWriter.JsonObjectState.InArrayContext);
Contract.Requires(!jsonBinaryWriter.JsonObjectState.InObjectContext);
Contract.Requires(!jsonBinaryWriter.JsonObjectState.IsPropertyExpected);
Expand Down Expand Up @@ -243,13 +243,6 @@ private enum RawValueType : byte
/// </summary>
private readonly Stack<ArrayAndObjectInfo> bufferedContexts;

/// <summary>
/// With binary encoding json elements like arrays and object are prefixed with a length in bytes and optionally a count.
/// This flag just determines whether you want to serialize the count, since it's optional and up to the user to make the
/// tradeoff between O(1) .Count() operation as the cost of additional storage.
/// </summary>
private readonly bool serializeCount;

/// <summary>
/// When a user writes an open array or object we reserve this much space for the type marker + length + count
/// And correct it later when they write a close array or object.
Expand All @@ -269,18 +262,15 @@ private enum RawValueType : byte
/// Initializes a new instance of the JsonBinaryWriter class.
/// </summary>
/// <param name="initialCapacity">The initial capacity to avoid intermediary allocations.</param>
/// <param name="serializeCount">Whether to serialize the count for object and array typemarkers.</param>
/// <param name="enableEncodedStrings">enable reference string encoding</param>
public JsonBinaryWriter(
int initialCapacity,
bool serializeCount,
bool enableEncodedStrings)
{
this.EnableEncodedStrings = enableEncodedStrings;
this.binaryWriter = new JsonBinaryMemoryWriter(initialCapacity);
this.bufferedContexts = new Stack<ArrayAndObjectInfo>();
this.serializeCount = serializeCount;
this.reservationSize = JsonBinaryEncoding.TypeMarkerLength + JsonBinaryEncoding.OneByteLength + (this.serializeCount ? JsonBinaryEncoding.OneByteCount : 0);
this.reservationSize = JsonBinaryEncoding.TypeMarkerLength + JsonBinaryEncoding.OneByteLength;
this.sharedStrings = new List<SharedStringValue>();
this.sharedStringIndexes = new ReferenceStringDictionary();
this.stringReferenceOffsets = new List<int>();
Expand All @@ -300,13 +290,13 @@ public JsonBinaryWriter(
public override long CurrentLength => this.binaryWriter.Position;

/// <inheritdoc />
public override void WriteObjectStart() => this.WriterArrayOrObjectStart(isArray: false);
public override void WriteObjectStart() => this.WriteArrayOrObjectStart(isArray: false);

/// <inheritdoc />
public override void WriteObjectEnd() => this.WriteArrayOrObjectEnd(isArray: false);

/// <inheritdoc />
public override void WriteArrayStart() => this.WriterArrayOrObjectStart(isArray: true);
public override void WriteArrayStart() => this.WriteArrayOrObjectStart(isArray: true);

/// <inheritdoc />
public override void WriteArrayEnd() => this.WriteArrayOrObjectEnd(isArray: true);
Expand Down Expand Up @@ -536,18 +526,14 @@ internal PreblittedBinaryJsonScope CapturePreblittedBinaryJsonScope(int startPos
this.binaryWriter.BufferAsSpan.Slice(startPosition, this.binaryWriter.Position - startPosition).ToArray());
}

private void WriterArrayOrObjectStart(bool isArray)
private void WriteArrayOrObjectStart(bool isArray)
{
this.RegisterArrayOrObjectStart(isArray, this.binaryWriter.Position, valueCount: 0);

// Assume 1-byte value length; as such, we need to reserve up 3 bytes (1 byte type marker, 1 byte length, 1 byte count).
// We'll adjust this as needed when writing the end of the array/object.
this.binaryWriter.Write((byte)0);
this.binaryWriter.Write((byte)0);
if (this.serializeCount)
{
this.binaryWriter.Write((byte)0);
}
}

private void RegisterArrayOrObjectStart(bool isArray, long offset, int valueCount)
Expand Down Expand Up @@ -611,16 +597,20 @@ private void WriteArrayOrObjectEnd(bool isArray)
// Need to figure out how many bytes to encode the length and the count
if (payloadLength <= byte.MaxValue)
{
// 1 byte length - don't need to move the buffer
bool serializeCount = isArray && (count > 16);

// 1 byte length - move the buffer forward
Span<byte> buffer = this.binaryWriter.BufferAsSpan;
int bytesToWrite = JsonBinaryEncoding.TypeMarkerLength
+ JsonBinaryEncoding.OneByteLength
+ (this.serializeCount ? JsonBinaryEncoding.OneByteCount : 0);
+ (serializeCount ? JsonBinaryEncoding.OneByteCount : 0);
this.MoveBuffer(buffer, payloadIndex, payloadLength, typeMarkerIndex, bytesToWrite, stringStartIndex, stringReferenceStartIndex);

// Move the cursor back
this.binaryWriter.Position = typeMarkerIndex;

// Write the type marker
if (this.serializeCount)
if (serializeCount)
{
this.binaryWriter.Write(isArray ? JsonBinaryEncoding.TypeMarker.Array1ByteLengthAndCount : JsonBinaryEncoding.TypeMarker.Object1ByteLengthAndCount);
this.binaryWriter.Write((byte)payloadLength);
Expand All @@ -637,9 +627,11 @@ private void WriteArrayOrObjectEnd(bool isArray)
}
else if (payloadLength <= ushort.MaxValue)
{
bool serializeCount = isArray && ((count > 16) || (payloadLength > 0x1000));

// 2 byte length - make space for the extra byte length (and extra byte count)
this.binaryWriter.Write((byte)0);
if (this.serializeCount)
if (serializeCount)
{
this.binaryWriter.Write((byte)0);
}
Expand All @@ -648,14 +640,14 @@ private void WriteArrayOrObjectEnd(bool isArray)
Span<byte> buffer = this.binaryWriter.BufferAsSpan;
int bytesToWrite = JsonBinaryEncoding.TypeMarkerLength
+ JsonBinaryEncoding.TwoByteLength
+ (this.serializeCount ? JsonBinaryEncoding.TwoByteCount : 0);
+ (serializeCount ? JsonBinaryEncoding.TwoByteCount : 0);
this.MoveBuffer(buffer, payloadIndex, payloadLength, typeMarkerIndex, bytesToWrite, stringStartIndex, stringReferenceStartIndex);

// Move the cursor back
this.binaryWriter.Position = typeMarkerIndex;

// Write the type marker
if (this.serializeCount)
if (serializeCount)
{
this.binaryWriter.Write(isArray ? JsonBinaryEncoding.TypeMarker.Array2ByteLengthAndCount : JsonBinaryEncoding.TypeMarker.Object2ByteLengthAndCount);
this.binaryWriter.Write((ushort)payloadLength);
Expand All @@ -673,11 +665,12 @@ private void WriteArrayOrObjectEnd(bool isArray)
else
{
// (payloadLength <= uint.MaxValue)
bool serializeCount = isArray;

// 4 byte length - make space for an extra 3 byte length (and 3 byte count)
this.binaryWriter.Write((byte)0);
this.binaryWriter.Write((ushort)0);
if (this.serializeCount)
if (serializeCount)
{
this.binaryWriter.Write((byte)0);
this.binaryWriter.Write((ushort)0);
Expand All @@ -687,14 +680,14 @@ private void WriteArrayOrObjectEnd(bool isArray)
Span<byte> buffer = this.binaryWriter.BufferAsSpan;
int bytesToWrite = JsonBinaryEncoding.TypeMarkerLength
+ JsonBinaryEncoding.FourByteLength
+ (this.serializeCount ? JsonBinaryEncoding.FourByteCount : 0);
+ (serializeCount ? JsonBinaryEncoding.FourByteCount : 0);
this.MoveBuffer(buffer, payloadIndex, payloadLength, typeMarkerIndex, bytesToWrite, stringStartIndex, stringReferenceStartIndex);

// Move the cursor back
this.binaryWriter.Position = typeMarkerIndex;

// Write the type marker
if (this.serializeCount)
if (serializeCount)
{
this.binaryWriter.Write(isArray ? JsonBinaryEncoding.TypeMarker.Array4ByteLengthAndCount : JsonBinaryEncoding.TypeMarker.Object4ByteLengthAndCount);
this.binaryWriter.Write((uint)payloadLength);
Expand Down Expand Up @@ -915,7 +908,7 @@ private bool TryRegisterStringValue(Utf8Span utf8Span)
// In order to avoid having to change the typer marker later on, we need to account for the case
// where the buffer might shift as a result of adjusting array/object length.

int maxOffset = (this.JsonObjectState.CurrentDepth * 3) + (int)this.CurrentLength;
int maxOffset = (this.JsonObjectState.CurrentDepth * 7) + (int)this.CurrentLength;

bool shouldAddValue = (utf8Span.Length >= 5) ||
((maxOffset <= byte.MaxValue) && (utf8Span.Length >= 2)) ||
Expand Down
1 change: 0 additions & 1 deletion Microsoft.Azure.Cosmos/src/Json/JsonWriter.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@ public static IJsonWriter Create(
JsonSerializationFormat.Text => new JsonTextWriter(initalCapacity),
JsonSerializationFormat.Binary => new JsonBinaryWriter(
initialCapacity: initalCapacity,
serializeCount: false,
enableEncodedStrings: enableEncodedStrings),
_ => throw new ArgumentException(
string.Format(
Expand Down
Loading