Skip to content

Commit

Permalink
slicer: Reuse stub object
Browse files Browse the repository at this point in the history
Updated after benchmark. During object slicing we create object for each part, we may reuse it, just resetting some fields.

                                                 │   old.txt   │               new.txt               │
                                                 │   sec/op    │    sec/op     vs base               │
SliceDataIntoObjects/slice_1-128/reader-16         62.90µ ± 0%   62.77µ ±  0%        ~ (p=0.589 n=6)
SliceDataIntoObjects/slice_1-128/writer-16         65.30µ ± 1%   65.82µ ±  0%   +0.80% (p=0.002 n=6)
SliceDataIntoObjects/slice_4-128/reader-16         61.19µ ± 1%   67.40µ ±  1%  +10.16% (p=0.002 n=6)
SliceDataIntoObjects/slice_4-128/writer-16         63.62µ ± 1%   70.42µ ±  2%  +10.69% (p=0.002 n=6)
SliceDataIntoObjects/slice_16-128/reader-16        63.15µ ± 1%   65.74µ ±  1%   +4.10% (p=0.002 n=6)
SliceDataIntoObjects/slice_16-128/writer-16        64.99µ ± 1%   68.16µ ±  1%   +4.87% (p=0.002 n=6)
SliceDataIntoObjects/slice_64-128/reader-16        66.61µ ± 1%   66.09µ ±  1%        ~ (p=0.093 n=6)
SliceDataIntoObjects/slice_64-128/writer-16        71.96µ ± 2%   77.10µ ±  7%   +7.14% (p=0.041 n=6)
SliceDataIntoObjects/slice_256-128/reader-16       65.92µ ± 1%   70.01µ ± 10%        ~ (p=0.394 n=6)
SliceDataIntoObjects/slice_256-128/writer-16       252.7µ ± 1%   234.0µ ±  1%   -7.42% (p=0.002 n=6)
SliceDataIntoObjects/slice_1024-128/reader-16      64.30µ ± 2%   65.18µ ±  6%        ~ (p=0.240 n=6)
SliceDataIntoObjects/slice_1024-128/writer-16      611.3µ ± 2%   570.0µ ±  5%   -6.76% (p=0.002 n=6)
SliceDataIntoObjects/slice_4096-128/reader-16      67.79µ ± 1%   69.01µ ±  3%   +1.80% (p=0.002 n=6)
SliceDataIntoObjects/slice_4096-128/writer-16      2.073m ± 1%   2.024m ±  3%   -2.39% (p=0.026 n=6)
SliceDataIntoObjects/slice_16384-128/reader-16     64.50µ ± 1%   64.07µ ±  0%   -0.66% (p=0.026 n=6)
SliceDataIntoObjects/slice_16384-128/writer-16     7.067m ± 1%   6.792m ±  8%        ~ (p=0.065 n=6)
SliceDataIntoObjects/slice_65536-128/reader-16     65.98µ ± 1%   69.42µ ±  1%   +5.22% (p=0.002 n=6)
SliceDataIntoObjects/slice_65536-128/writer-16     31.06m ± 1%   30.11m ±  1%   -3.07% (p=0.002 n=6)
SliceDataIntoObjects/slice_262144-128/reader-16    73.88µ ± 1%   76.69µ ±  3%   +3.81% (p=0.002 n=6)
SliceDataIntoObjects/slice_262144-128/writer-16    112.6m ± 0%   123.6m ±  4%   +9.76% (p=0.002 n=6)
SliceDataIntoObjects/slice_1048576-128/reader-16   125.1µ ± 3%   106.8µ ± 12%  -14.61% (p=0.002 n=6)
SliceDataIntoObjects/slice_1048576-128/writer-16   510.0m ± 1%   444.4m ±  1%  -12.85% (p=0.002 n=6)
geomean                                            321.5µ        322.7µ         +0.37%

                                                 │    old.txt    │               new.txt                │
                                                 │     B/op      │     B/op       vs base               │
SliceDataIntoObjects/slice_1-128/reader-16          9.547Ki ± 0%    9.547Ki ± 0%        ~ (p=1.000 n=6)
SliceDataIntoObjects/slice_1-128/writer-16          9.438Ki ± 0%    9.421Ki ± 0%   -0.18% (p=0.002 n=6)
SliceDataIntoObjects/slice_4-128/reader-16          9.077Ki ± 0%   11.298Ki ± 0%  +24.46% (p=0.002 n=6)
SliceDataIntoObjects/slice_4-128/writer-16          8.968Ki ± 0%   11.173Ki ± 0%  +24.59% (p=0.002 n=6)
SliceDataIntoObjects/slice_16-128/reader-16         9.547Ki ± 0%   10.548Ki ± 0%  +10.48% (p=0.002 n=6)
SliceDataIntoObjects/slice_16-128/writer-16         9.438Ki ± 0%   10.423Ki ± 0%  +10.44% (p=0.002 n=6)
SliceDataIntoObjects/slice_64-128/reader-16         11.17Ki ± 0%    10.55Ki ± 0%   -5.59% (p=0.002 n=6)
SliceDataIntoObjects/slice_64-128/writer-16         11.06Ki ± 0%    10.42Ki ± 0%   -5.79% (p=0.002 n=6)
SliceDataIntoObjects/slice_256-128/reader-16       10.893Ki ± 0%    9.204Ki ± 0%  -15.50% (p=0.002 n=6)
SliceDataIntoObjects/slice_256-128/writer-16        38.08Ki ± 0%    29.42Ki ± 0%  -22.73% (p=0.002 n=6)
SliceDataIntoObjects/slice_1024-128/reader-16       9.425Ki ± 0%    9.675Ki ± 0%   +2.65% (p=0.002 n=6)
SliceDataIntoObjects/slice_1024-128/writer-16       75.66Ki ± 0%    69.60Ki ± 0%   -8.01% (p=0.002 n=6)
SliceDataIntoObjects/slice_4096-128/reader-16       11.19Ki ± 0%    11.31Ki ± 0%   +1.11% (p=0.002 n=6)
SliceDataIntoObjects/slice_4096-128/writer-16       260.8Ki ± 0%    235.7Ki ± 0%   -9.65% (p=0.002 n=6)
SliceDataIntoObjects/slice_16384-128/reader-16     10.004Ki ± 0%    9.716Ki ± 0%   -2.88% (p=0.002 n=6)
SliceDataIntoObjects/slice_16384-128/writer-16      936.3Ki ± 0%    831.3Ki ± 0%  -11.21% (p=0.002 n=6)
SliceDataIntoObjects/slice_65536-128/reader-16      10.10Ki ± 0%    11.50Ki ± 0%  +13.79% (p=0.002 n=6)
SliceDataIntoObjects/slice_65536-128/writer-16      3.708Mi ± 0%    3.307Mi ± 0%  -10.80% (p=0.002 n=6)
SliceDataIntoObjects/slice_262144-128/reader-16     11.92Ki ± 0%    12.32Ki ± 0%   +3.36% (p=0.002 n=6)
SliceDataIntoObjects/slice_262144-128/writer-16     14.45Mi ± 0%    13.26Mi ± 0%   -8.21% (p=0.002 n=6)
SliceDataIntoObjects/slice_1048576-128/reader-16    18.17Ki ± 3%    15.16Ki ± 6%  -16.53% (p=0.002 n=6)
SliceDataIntoObjects/slice_1048576-128/writer-16    59.48Mi ± 0%    51.22Mi ± 0%  -13.89% (p=0.002 n=6)
geomean                                             46.55Ki         45.35Ki        -2.56%

                                                 │   old.txt   │               new.txt                │
                                                 │  allocs/op  │  allocs/op   vs base                 │
SliceDataIntoObjects/slice_1-128/reader-16          149.0 ± 0%    149.0 ± 0%        ~ (p=1.000 n=6) ¹
SliceDataIntoObjects/slice_1-128/writer-16          148.0 ± 0%    148.0 ± 0%        ~ (p=1.000 n=6) ¹
SliceDataIntoObjects/slice_4-128/reader-16          142.0 ± 0%    174.0 ± 0%  +22.54% (p=0.002 n=6)
SliceDataIntoObjects/slice_4-128/writer-16          141.0 ± 0%    173.0 ± 0%  +22.70% (p=0.002 n=6)
SliceDataIntoObjects/slice_16-128/reader-16         149.0 ± 0%    163.0 ± 0%   +9.40% (p=0.002 n=6)
SliceDataIntoObjects/slice_16-128/writer-16         148.0 ± 0%    162.0 ± 0%   +9.46% (p=0.002 n=6)
SliceDataIntoObjects/slice_64-128/reader-16         172.0 ± 0%    163.0 ± 0%   -5.23% (p=0.002 n=6)
SliceDataIntoObjects/slice_64-128/writer-16         171.0 ± 0%    162.0 ± 0%   -5.26% (p=0.002 n=6)
SliceDataIntoObjects/slice_256-128/reader-16        167.0 ± 0%    144.0 ± 0%  -13.77% (p=0.002 n=6)
SliceDataIntoObjects/slice_256-128/writer-16        583.0 ± 0%    437.0 ± 0%  -25.04% (p=0.002 n=6)
SliceDataIntoObjects/slice_1024-128/reader-16       147.0 ± 0%    151.0 ± 0%   +2.72% (p=0.002 n=6)
SliceDataIntoObjects/slice_1024-128/writer-16      1.155k ± 0%   1.030k ± 0%  -10.82% (p=0.002 n=6)
SliceDataIntoObjects/slice_4096-128/reader-16       172.0 ± 0%    174.0 ± 0%   +1.16% (p=0.002 n=6)
SliceDataIntoObjects/slice_4096-128/writer-16      4.021k ± 0%   3.464k ± 0%  -13.85% (p=0.002 n=6)
SliceDataIntoObjects/slice_16384-128/reader-16      155.0 ± 0%    151.0 ± 0%   -2.58% (p=0.002 n=6)
SliceDataIntoObjects/slice_16384-128/writer-16     14.51k ± 0%   12.20k ± 0%  -15.96% (p=0.002 n=6)
SliceDataIntoObjects/slice_65536-128/reader-16      157.0 ± 0%    176.0 ± 0%  +12.10% (p=0.002 n=6)
SliceDataIntoObjects/slice_65536-128/writer-16     58.66k ± 0%   49.56k ± 0%  -15.52% (p=0.002 n=6)
SliceDataIntoObjects/slice_262144-128/reader-16     182.0 ± 0%    188.0 ± 1%   +3.30% (p=0.002 n=6)
SliceDataIntoObjects/slice_262144-128/writer-16    227.8k ± 0%   197.1k ± 0%  -13.48% (p=0.002 n=6)
SliceDataIntoObjects/slice_1048576-128/reader-16    279.0 ± 3%    230.5 ± 6%  -17.38% (p=0.002 n=6)
SliceDataIntoObjects/slice_1048576-128/writer-16   934.5k ± 0%   762.4k ± 0%  -18.42% (p=0.002 n=6)
geomean                                             720.3         690.1        -4.19%

Signed-off-by: Evgenii Baidakov <[email protected]>
  • Loading branch information
smallhive committed Jul 28, 2023
1 parent 8b7c80b commit d6858d9
Show file tree
Hide file tree
Showing 2 changed files with 37 additions and 12 deletions.
17 changes: 17 additions & 0 deletions object/object.go
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,14 @@ func (o *Object) SetID(v oid.ID) {
SetObjectID(&v2)
}

// ResetID removes object identifier.
//
// See also [Object.SetID].
func (o *Object) ResetID() {
(*object.Object)(o).
SetObjectID(nil)
}

// Signature returns signature of the object identifier.
//
// See also [Object.SetSignature].
Expand Down Expand Up @@ -536,6 +544,15 @@ func (o *Object) SetParentID(v oid.ID) {
})
}

// ResetParentID removes identifier of the parent object.
//
// See also [Object.SetParentID].
func (o *Object) ResetParentID() {
o.setSplitFields(func(split *object.SplitHeader) {
split.SetParent(nil)
})
}

// Parent returns parent object w/o payload.
//
// See also [Object.SetParent].
Expand Down
32 changes: 20 additions & 12 deletions object/slicer/slicer.go
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,14 @@ func initPayloadStream(ctx context.Context, ow ObjectWriter, header object.Objec
currentVersion := version.Current()
header.SetVersion(&currentVersion)

var stubObject object.Object
stubObject.SetVersion(&currentVersion)
stubObject.SetContainerID(containerID)
stubObject.SetCreationEpoch(opts.currentNeoFSEpoch)
stubObject.SetType(object.TypeRegular)
stubObject.SetOwnerID(&owner)
stubObject.SetSessionToken(opts.sessionToken)

res := &PayloadWriter{
ctx: ctx,
isHeaderWriteStep: true,
Expand All @@ -245,6 +253,7 @@ func initPayloadStream(ctx context.Context, ow ObjectWriter, header object.Objec
rootMeta: newDynamicObjectMetadata(opts.withHomoChecksum),
childMeta: newDynamicObjectMetadata(opts.withHomoChecksum),
prmObjectPutInit: prm,
stubObject: &stubObject,
}

maxObjSize := childPayloadSizeLimit(opts)
Expand Down Expand Up @@ -283,6 +292,7 @@ type PayloadWriter struct {

writtenChildren []oid.ID
prmObjectPutInit client.PrmObjectPutInit
stubObject *object.Object
}

// Write writes next chunk of the object data. Concatenation of all chunks forms
Expand Down Expand Up @@ -364,19 +374,14 @@ func (x *PayloadWriter) writeLastChild(ctx context.Context, meta dynamicObjectMe
}

func (x *PayloadWriter) _writeChild(ctx context.Context, meta dynamicObjectMetadata, last bool, rootIDHandler func(id oid.ID)) error {
currentVersion := version.Current()

fCommon := func(obj *object.Object) {
obj.SetVersion(&currentVersion)
obj.SetContainerID(x.container)
obj.SetCreationEpoch(x.currentEpoch)
obj.SetType(object.TypeRegular)
obj.SetOwnerID(&x.owner)
obj.SetSessionToken(x.sessionToken)
}
obj := *x.stubObject
obj.SetSplitID(nil)
obj.ResetPreviousID()
obj.SetParent(nil)
obj.ResetParentID()
obj.SetSignature(nil)
obj.ResetID()

var obj object.Object
fCommon(&obj)
if x.withSplit {
obj.SetSplitID(x.splitID)
}
Expand Down Expand Up @@ -420,6 +425,9 @@ func (x *PayloadWriter) _writeChild(ctx context.Context, meta dynamicObjectMetad
meta.reset()
obj.ResetPreviousID()
obj.SetChildren(x.writtenChildren...)
// we reuse already written object, we should reset these fields, to eval them one more time in writeInMemObject.
obj.ResetID()
obj.SetSignature(nil)

_, err = writeInMemObject(ctx, x.signer, x.stream, obj, nil, meta, x.prmObjectPutInit)
if err != nil {
Expand Down

0 comments on commit d6858d9

Please sign in to comment.