From 63bae2d12e0168f88aaf572f482e90f70ca92854 Mon Sep 17 00:00:00 2001 From: Calvin Kim Date: Mon, 1 Feb 2021 21:58:54 +0900 Subject: [PATCH] wire/common: Implement binarySerializer based on sync.Pool --- wire/common.go | 312 ++++++++++++++++++++++++--------------------- wire/msgtx.go | 33 +++-- wire/netaddress.go | 4 +- 3 files changed, 195 insertions(+), 154 deletions(-) diff --git a/wire/common.go b/wire/common.go index 42c1797b32..af3c099f19 100644 --- a/wire/common.go +++ b/wire/common.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "math" + "sync" "time" "github.com/btcsuite/btcd/chaincfg/chainhash" @@ -18,10 +19,6 @@ import ( const ( // MaxVarIntPayload is the maximum payload size for a variable length integer. MaxVarIntPayload = 9 - - // binaryFreeListMaxItems is the number of buffers to keep in the free - // list to use for binary serialization and deserialization. - binaryFreeListMaxItems = 1024 ) var ( @@ -34,143 +31,117 @@ var ( bigEndian = binary.BigEndian ) -// binaryFreeList defines a concurrent safe free list of byte slices (up to the -// maximum number defined by the binaryFreeListMaxItems constant) that have a -// cap of 8 (thus it supports up to a uint64). It is used to provide temporary -// buffers for serializing and deserializing primitive numbers to and from their -// binary encoding in order to greatly reduce the number of allocations -// required. -// -// For convenience, functions are provided for each of the primitive unsigned -// integers that automatically obtain a buffer from the free list, perform the -// necessary binary conversion, read from or write to the given io.Reader or -// io.Writer, and return the buffer to the free list. -type binaryFreeList chan []byte - -// Borrow returns a byte slice from the free list with a length of 8. A new -// buffer is allocated if there are not any available on the free list. -func (l binaryFreeList) Borrow() []byte { - var buf []byte - select { - case buf = <-l: - default: - buf = make([]byte, 8) - } - return buf[:8] +// binarySerializer is just a wrapper around a slice of bytes +type binarySerializer struct { + buf []byte } -// Return puts the provided byte slice back on the free list. The buffer MUST -// have been obtained via the Borrow function and therefore have a cap of 8. -func (l binaryFreeList) Return(buf []byte) { - select { - case l <- buf: - default: - // Let it go to the garbage collector. +// binarySerializerFree provides a free list of buffers to use for serializing and +// deserializing primitive integer values to and from io.Readers and io.Writers. +var binarySerializerFree = sync.Pool{ + New: func() interface{} { return new(binarySerializer) }, +} + +// newSerializer allocates a new binarySerializer struct or grabs a cached one +// from binarySerializerFree +func newSerializer() *binarySerializer { + b := binarySerializerFree.Get().(*binarySerializer) + + if b.buf == nil { + b.buf = make([]byte, 8) } + + return b +} + +// free saves used binarySerializer structs in ppFree; avoids an allocation per invocation. +func (bs *binarySerializer) free() { + bs.buf = bs.buf[:0] + binarySerializerFree.Put(bs) } // Uint8 reads a single byte from the provided reader using a buffer from the // free list and returns it as a uint8. -func (l binaryFreeList) Uint8(r io.Reader) (uint8, error) { - buf := l.Borrow()[:1] +func (l *binarySerializer) Uint8(r io.Reader) (uint8, error) { + buf := l.buf[:1] if _, err := io.ReadFull(r, buf); err != nil { - l.Return(buf) return 0, err } - rv := buf[0] - l.Return(buf) - return rv, nil + return buf[0], nil } // Uint16 reads two bytes from the provided reader using a buffer from the // free list, converts it to a number using the provided byte order, and returns // the resulting uint16. -func (l binaryFreeList) Uint16(r io.Reader, byteOrder binary.ByteOrder) (uint16, error) { - buf := l.Borrow()[:2] +func (l *binarySerializer) Uint16(r io.Reader, byteOrder binary.ByteOrder) (uint16, error) { + buf := l.buf[:2] if _, err := io.ReadFull(r, buf); err != nil { - l.Return(buf) return 0, err } - rv := byteOrder.Uint16(buf) - l.Return(buf) - return rv, nil + return byteOrder.Uint16(buf), nil } // Uint32 reads four bytes from the provided reader using a buffer from the // free list, converts it to a number using the provided byte order, and returns // the resulting uint32. -func (l binaryFreeList) Uint32(r io.Reader, byteOrder binary.ByteOrder) (uint32, error) { - buf := l.Borrow()[:4] +func (l *binarySerializer) Uint32(r io.Reader, byteOrder binary.ByteOrder) (uint32, error) { + buf := l.buf[:4] if _, err := io.ReadFull(r, buf); err != nil { - l.Return(buf) return 0, err } - rv := byteOrder.Uint32(buf) - l.Return(buf) - return rv, nil + return byteOrder.Uint32(buf), nil } // Uint64 reads eight bytes from the provided reader using a buffer from the // free list, converts it to a number using the provided byte order, and returns // the resulting uint64. -func (l binaryFreeList) Uint64(r io.Reader, byteOrder binary.ByteOrder) (uint64, error) { - buf := l.Borrow()[:8] +func (l *binarySerializer) Uint64(r io.Reader, byteOrder binary.ByteOrder) (uint64, error) { + buf := l.buf[:8] if _, err := io.ReadFull(r, buf); err != nil { - l.Return(buf) return 0, err } - rv := byteOrder.Uint64(buf) - l.Return(buf) - return rv, nil + return byteOrder.Uint64(buf), nil } // PutUint8 copies the provided uint8 into a buffer from the free list and // writes the resulting byte to the given writer. -func (l binaryFreeList) PutUint8(w io.Writer, val uint8) error { - buf := l.Borrow()[:1] +func (l *binarySerializer) PutUint8(w io.Writer, val uint8) error { + buf := l.buf[:1] buf[0] = val _, err := w.Write(buf) - l.Return(buf) return err } // PutUint16 serializes the provided uint16 using the given byte order into a // buffer from the free list and writes the resulting two bytes to the given // writer. -func (l binaryFreeList) PutUint16(w io.Writer, byteOrder binary.ByteOrder, val uint16) error { - buf := l.Borrow()[:2] +func (l *binarySerializer) PutUint16(w io.Writer, byteOrder binary.ByteOrder, val uint16) error { + buf := l.buf[:2] byteOrder.PutUint16(buf, val) _, err := w.Write(buf) - l.Return(buf) return err } // PutUint32 serializes the provided uint32 using the given byte order into a // buffer from the free list and writes the resulting four bytes to the given // writer. -func (l binaryFreeList) PutUint32(w io.Writer, byteOrder binary.ByteOrder, val uint32) error { - buf := l.Borrow()[:4] +func (l *binarySerializer) PutUint32(w io.Writer, byteOrder binary.ByteOrder, val uint32) error { + buf := l.buf[:4] byteOrder.PutUint32(buf, val) _, err := w.Write(buf) - l.Return(buf) return err } // PutUint64 serializes the provided uint64 using the given byte order into a // buffer from the free list and writes the resulting eight bytes to the given // writer. -func (l binaryFreeList) PutUint64(w io.Writer, byteOrder binary.ByteOrder, val uint64) error { - buf := l.Borrow()[:8] +func (l *binarySerializer) PutUint64(w io.Writer, byteOrder binary.ByteOrder, val uint64) error { + buf := l.buf[:8] byteOrder.PutUint64(buf, val) _, err := w.Write(buf) - l.Return(buf) return err } -// binarySerializer provides a free list of buffers to use for serializing and -// deserializing primitive integer values to and from io.Readers and io.Writers. -var binarySerializer binaryFreeList = make(chan []byte, binaryFreeListMaxItems) - // errNonCanonicalVarInt is the common format string used for non-canonically // encoded variable length integer errors. var errNonCanonicalVarInt = "non-canonical varint %x - discriminant %x must " + @@ -193,7 +164,9 @@ func readElement(r io.Reader, element interface{}) error { // type assertions first. switch e := element.(type) { case *int32: - rv, err := binarySerializer.Uint32(r, littleEndian) + bs := newSerializer() + rv, err := bs.Uint32(r, littleEndian) + bs.free() if err != nil { return err } @@ -201,7 +174,9 @@ func readElement(r io.Reader, element interface{}) error { return nil case *uint32: - rv, err := binarySerializer.Uint32(r, littleEndian) + bs := newSerializer() + rv, err := bs.Uint32(r, littleEndian) + bs.free() if err != nil { return err } @@ -209,7 +184,9 @@ func readElement(r io.Reader, element interface{}) error { return nil case *int64: - rv, err := binarySerializer.Uint64(r, littleEndian) + bs := newSerializer() + rv, err := bs.Uint64(r, littleEndian) + bs.free() if err != nil { return err } @@ -217,7 +194,9 @@ func readElement(r io.Reader, element interface{}) error { return nil case *uint64: - rv, err := binarySerializer.Uint64(r, littleEndian) + bs := newSerializer() + rv, err := bs.Uint64(r, littleEndian) + bs.free() if err != nil { return err } @@ -225,7 +204,9 @@ func readElement(r io.Reader, element interface{}) error { return nil case *bool: - rv, err := binarySerializer.Uint8(r) + bs := newSerializer() + rv, err := bs.Uint8(r) + bs.free() if err != nil { return err } @@ -238,7 +219,9 @@ func readElement(r io.Reader, element interface{}) error { // Unix timestamp encoded as a uint32. case *uint32Time: - rv, err := binarySerializer.Uint32(r, binary.LittleEndian) + bs := newSerializer() + rv, err := bs.Uint32(r, binary.LittleEndian) + bs.free() if err != nil { return err } @@ -247,7 +230,9 @@ func readElement(r io.Reader, element interface{}) error { // Unix timestamp encoded as an int64. case *int64Time: - rv, err := binarySerializer.Uint64(r, binary.LittleEndian) + bs := newSerializer() + rv, err := bs.Uint64(r, binary.LittleEndian) + bs.free() if err != nil { return err } @@ -286,7 +271,9 @@ func readElement(r io.Reader, element interface{}) error { return nil case *ServiceFlag: - rv, err := binarySerializer.Uint64(r, littleEndian) + bs := newSerializer() + rv, err := bs.Uint64(r, littleEndian) + bs.free() if err != nil { return err } @@ -294,7 +281,9 @@ func readElement(r io.Reader, element interface{}) error { return nil case *InvType: - rv, err := binarySerializer.Uint32(r, littleEndian) + bs := newSerializer() + rv, err := bs.Uint32(r, littleEndian) + bs.free() if err != nil { return err } @@ -302,7 +291,9 @@ func readElement(r io.Reader, element interface{}) error { return nil case *BitcoinNet: - rv, err := binarySerializer.Uint32(r, littleEndian) + bs := newSerializer() + rv, err := bs.Uint32(r, littleEndian) + bs.free() if err != nil { return err } @@ -310,7 +301,9 @@ func readElement(r io.Reader, element interface{}) error { return nil case *BloomUpdateType: - rv, err := binarySerializer.Uint8(r) + bs := newSerializer() + rv, err := bs.Uint8(r) + bs.free() if err != nil { return err } @@ -318,7 +311,9 @@ func readElement(r io.Reader, element interface{}) error { return nil case *RejectCode: - rv, err := binarySerializer.Uint8(r) + bs := newSerializer() + rv, err := bs.Uint8(r) + bs.free() if err != nil { return err } @@ -349,39 +344,45 @@ func writeElement(w io.Writer, element interface{}) error { // type assertions first. switch e := element.(type) { case int32: - err := binarySerializer.PutUint32(w, littleEndian, uint32(e)) - if err != nil { - return err - } - return nil + bs := newSerializer() + err := bs.PutUint32(w, littleEndian, uint32(e)) + bs.free() + return err case uint32: - err := binarySerializer.PutUint32(w, littleEndian, e) - if err != nil { - return err - } - return nil + bs := newSerializer() + err := bs.PutUint32(w, littleEndian, e) + bs.free() + return err case int64: - err := binarySerializer.PutUint64(w, littleEndian, uint64(e)) - if err != nil { - return err - } - return nil + bs := newSerializer() + err := bs.PutUint64(w, littleEndian, uint64(e)) + bs.free() + return err case uint64: - err := binarySerializer.PutUint64(w, littleEndian, e) - if err != nil { - return err - } - return nil + bs := newSerializer() + err := bs.PutUint64(w, littleEndian, e) + bs.free() + return err case bool: var err error if e { - err = binarySerializer.PutUint8(w, 0x01) + bs := newSerializer() + err = bs.PutUint8(w, 0x01) + bs.free() + if err != nil { + return err + } } else { - err = binarySerializer.PutUint8(w, 0x00) + bs := newSerializer() + err = bs.PutUint8(w, 0x00) + bs.free() + if err != nil { + return err + } } if err != nil { return err @@ -420,39 +421,34 @@ func writeElement(w io.Writer, element interface{}) error { return nil case ServiceFlag: - err := binarySerializer.PutUint64(w, littleEndian, uint64(e)) - if err != nil { - return err - } - return nil + bs := newSerializer() + err := bs.PutUint64(w, littleEndian, uint64(e)) + bs.free() + return err case InvType: - err := binarySerializer.PutUint32(w, littleEndian, uint32(e)) - if err != nil { - return err - } - return nil + bs := newSerializer() + err := bs.PutUint32(w, littleEndian, uint32(e)) + bs.free() + return err case BitcoinNet: - err := binarySerializer.PutUint32(w, littleEndian, uint32(e)) - if err != nil { - return err - } - return nil + bs := newSerializer() + err := bs.PutUint32(w, littleEndian, uint32(e)) + bs.free() + return err case BloomUpdateType: - err := binarySerializer.PutUint8(w, uint8(e)) - if err != nil { - return err - } - return nil + bs := newSerializer() + err := bs.PutUint8(w, uint8(e)) + bs.free() + return err case RejectCode: - err := binarySerializer.PutUint8(w, uint8(e)) - if err != nil { - return err - } - return nil + bs := newSerializer() + err := bs.PutUint8(w, uint8(e)) + bs.free() + return err } // Fall back to the slower binary.Write if a fast path was not available @@ -474,7 +470,9 @@ func writeElements(w io.Writer, elements ...interface{}) error { // ReadVarInt reads a variable length integer from r and returns it as a uint64. func ReadVarInt(r io.Reader, pver uint32) (uint64, error) { - discriminant, err := binarySerializer.Uint8(r) + bs := newSerializer() + discriminant, err := bs.Uint8(r) + bs.free() if err != nil { return 0, err } @@ -482,7 +480,9 @@ func ReadVarInt(r io.Reader, pver uint32) (uint64, error) { var rv uint64 switch discriminant { case 0xff: - sv, err := binarySerializer.Uint64(r, littleEndian) + bs := newSerializer() + sv, err := bs.Uint64(r, littleEndian) + bs.free() if err != nil { return 0, err } @@ -497,7 +497,9 @@ func ReadVarInt(r io.Reader, pver uint32) (uint64, error) { } case 0xfe: - sv, err := binarySerializer.Uint32(r, littleEndian) + bs := newSerializer() + sv, err := bs.Uint32(r, littleEndian) + bs.free() if err != nil { return 0, err } @@ -512,7 +514,9 @@ func ReadVarInt(r io.Reader, pver uint32) (uint64, error) { } case 0xfd: - sv, err := binarySerializer.Uint16(r, littleEndian) + bs := newSerializer() + sv, err := bs.Uint16(r, littleEndian) + bs.free() if err != nil { return 0, err } @@ -537,30 +541,46 @@ func ReadVarInt(r io.Reader, pver uint32) (uint64, error) { // on its value. func WriteVarInt(w io.Writer, pver uint32, val uint64) error { if val < 0xfd { - return binarySerializer.PutUint8(w, uint8(val)) + bs := newSerializer() + err := bs.PutUint8(w, uint8(val)) + bs.free() + return err } if val <= math.MaxUint16 { - err := binarySerializer.PutUint8(w, 0xfd) + bs := newSerializer() + err := bs.PutUint8(w, 0xfd) if err != nil { + bs.free() return err } - return binarySerializer.PutUint16(w, littleEndian, uint16(val)) + + err = bs.PutUint16(w, littleEndian, uint16(val)) + bs.free() + return err } if val <= math.MaxUint32 { - err := binarySerializer.PutUint8(w, 0xfe) + bs := newSerializer() + err := bs.PutUint8(w, 0xfe) if err != nil { + bs.free() return err } - return binarySerializer.PutUint32(w, littleEndian, uint32(val)) + err = bs.PutUint32(w, littleEndian, uint32(val)) + bs.free() + return err } - err := binarySerializer.PutUint8(w, 0xff) + bs := newSerializer() + err := bs.PutUint8(w, 0xff) if err != nil { + bs.free() return err } - return binarySerializer.PutUint64(w, littleEndian, val) + err = bs.PutUint64(w, littleEndian, val) + bs.free() + return err } // VarIntSerializeSize returns the number of bytes it would take to serialize @@ -676,7 +696,9 @@ func WriteVarBytes(w io.Writer, pver uint32, bytes []byte) error { // unexported version takes a reader primarily to ensure the error paths // can be properly tested by passing a fake reader in the tests. func randomUint64(r io.Reader) (uint64, error) { - rv, err := binarySerializer.Uint64(r, bigEndian) + bs := newSerializer() + rv, err := bs.Uint64(r, bigEndian) + bs.free() if err != nil { return 0, err } diff --git a/wire/msgtx.go b/wire/msgtx.go index 1e2f69fad4..408809264f 100644 --- a/wire/msgtx.go +++ b/wire/msgtx.go @@ -433,8 +433,10 @@ func (msg *MsgTx) Copy() *MsgTx { // See Deserialize for decoding transactions stored to disk, such as in a // database, as opposed to decoding transactions from the wire. func (msg *MsgTx) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error { - version, err := binarySerializer.Uint32(r, littleEndian) + bs := newSerializer() + version, err := bs.Uint32(r, littleEndian) if err != nil { + bs.free() return err } msg.Version = int32(version) @@ -597,7 +599,8 @@ func (msg *MsgTx) BtcDecode(r io.Reader, pver uint32, enc MessageEncoding) error } } - msg.LockTime, err = binarySerializer.Uint32(r, littleEndian) + msg.LockTime, err = bs.Uint32(r, littleEndian) + bs.free() if err != nil { returnScriptBuffers() return err @@ -704,8 +707,10 @@ func (msg *MsgTx) DeserializeNoWitness(r io.Reader) error { // See Serialize for encoding transactions to be stored to disk, such as in a // database, as opposed to encoding transactions for the wire. func (msg *MsgTx) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error { - err := binarySerializer.PutUint32(w, littleEndian, uint32(msg.Version)) + bs := newSerializer() + err := bs.PutUint32(w, littleEndian, uint32(msg.Version)) if err != nil { + bs.free() return err } @@ -762,7 +767,9 @@ func (msg *MsgTx) BtcEncode(w io.Writer, pver uint32, enc MessageEncoding) error } } - return binarySerializer.PutUint32(w, littleEndian, msg.LockTime) + err = bs.PutUint32(w, littleEndian, msg.LockTime) + bs.free() + return err } // HasWitness returns false if none of the inputs within the transaction @@ -926,7 +933,9 @@ func readOutPoint(r io.Reader, pver uint32, version int32, op *OutPoint) error { return err } - op.Index, err = binarySerializer.Uint32(r, littleEndian) + bs := newSerializer() + op.Index, err = bs.Uint32(r, littleEndian) + bs.free() return err } @@ -938,7 +947,10 @@ func writeOutPoint(w io.Writer, pver uint32, version int32, op *OutPoint) error return err } - return binarySerializer.PutUint32(w, littleEndian, op.Index) + bs := newSerializer() + err = bs.PutUint32(w, littleEndian, op.Index) + bs.free() + return err } // readScript reads a variable length byte array that represents a transaction @@ -1002,7 +1014,10 @@ func writeTxIn(w io.Writer, pver uint32, version int32, ti *TxIn) error { return err } - return binarySerializer.PutUint32(w, littleEndian, ti.Sequence) + bs := newSerializer() + err = bs.PutUint32(w, littleEndian, ti.Sequence) + bs.free() + return err } // readTxOut reads the next sequence of bytes from r as a transaction output @@ -1024,7 +1039,9 @@ func readTxOut(r io.Reader, pver uint32, version int32, to *TxOut) error { // NOTE: This function is exported in order to allow txscript to compute the // new sighashes for witness transactions (BIP0143). func WriteTxOut(w io.Writer, pver uint32, version int32, to *TxOut) error { - err := binarySerializer.PutUint64(w, littleEndian, uint64(to.Value)) + bs := newSerializer() + err := bs.PutUint64(w, littleEndian, uint64(to.Value)) + bs.free() if err != nil { return err } diff --git a/wire/netaddress.go b/wire/netaddress.go index 5a2610bccc..3129631095 100644 --- a/wire/netaddress.go +++ b/wire/netaddress.go @@ -106,7 +106,9 @@ func readNetAddress(r io.Reader, pver uint32, na *NetAddress, ts bool) error { return err } // Sigh. Bitcoin protocol mixes little and big endian. - port, err := binarySerializer.Uint16(r, bigEndian) + bs := newSerializer() + port, err := bs.Uint16(r, bigEndian) + bs.free() if err != nil { return err }