From 01ef9cdcd674deb77fa8dd49be149af139d15f90 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 22 Mar 2023 11:02:42 -0400 Subject: [PATCH] conversion: add `PrettyDec`, optimize `Dec`, add benchmarks and fuzzing (#130) This PR implements the Dec method 'natively', and adds the API-method PrettyDec. // Dec returns the decimal representation of z. func (z *Int) Dec() string // PrettyDec returns the decimal representation of z, with thousands-separators. func (z *Int) PrettyDec(separator byte) string It also adds benchmarks and a new fuzz-test for the changed methods. --- circle.yml | 3 +- conversion.go | 2 +- conversion_test.go | 187 ++++++++++++++++++++++++++++++++++++++++----- decimal.go | 97 ++++++++++++++++++++++- 4 files changed, 269 insertions(+), 20 deletions(-) diff --git a/circle.yml b/circle.yml index c6047ee0..9b56ce82 100644 --- a/circle.yml +++ b/circle.yml @@ -44,7 +44,8 @@ jobs: - run: name: "Fuzzing" command: | - GOCACHE=/home/circleci/project/corpus-v3 go test . -run - -fuzz . -fuzztime 1m + GOCACHE=/home/circleci/project/corpus-v3 go test . -run - -fuzz FuzzBase10StringCompare -fuzztime 30s + GOCACHE=/home/circleci/project/corpus-v3 go test . -run - -fuzz FuzzDecimal -fuzztime 30s - save_cache: key: corpus-v3-{{ epoch }} paths: diff --git a/conversion.go b/conversion.go index b209433f..aa7c5a9f 100644 --- a/conversion.go +++ b/conversion.go @@ -667,7 +667,7 @@ func (dst *Int) scanScientificFromString(src string) error { // In MariaDB/MySQL, this will work with the Numeric/Decimal types up to 65 digits, however any more and you should use either VarChar or Char(79) // In SqLite, use TEXT func (src *Int) Value() (driver.Value, error) { - return src.ToBig().String(), nil + return src.Dec(), nil } var ( diff --git a/conversion_test.go b/conversion_test.go index 822e5a52..58b35d7d 100644 --- a/conversion_test.go +++ b/conversion_test.go @@ -1157,7 +1157,6 @@ func TestEncode(t *testing.T) { t.Errorf("input %x: wrong encoding %s (exp %s)", test.input, enc, test.want) } } - } func TestDecode(t *testing.T) { @@ -1214,18 +1213,21 @@ func TestEnDecode(t *testing.T) { if got, _ := intSample.Value(); wantDec != got.(string) { t.Fatalf("test %d #4, got %v, exp %v", i, got, wantHex) } + if got := intSample.Dec(); wantDec != got { + t.Fatalf("test %d #5, got %v, exp %v", i, got, wantHex) + } { // Json jsonEncoded, err := json.Marshal(&jsonStruct{&intSample}) if err != nil { - t.Fatalf("test %d #4, err: %v", i, err) + t.Fatalf("test %d #6, err: %v", i, err) } var jsonDecoded jsonStruct err = json.Unmarshal(jsonEncoded, &jsonDecoded) if err != nil { - t.Fatalf("test %d #5, err: %v", i, err) + t.Fatalf("test %d #7, err: %v", i, err) } if jsonDecoded.Foo.Cmp(&intSample) != 0 { - t.Fatalf("test %d #6, got %v, exp %v", i, jsonDecoded.Foo, intSample) + t.Fatalf("test %d #8, got %v, exp %v", i, jsonDecoded.Foo, intSample) } } // Decoding @@ -1234,67 +1236,67 @@ func TestEnDecode(t *testing.T) { decoded, err := FromHex(wantHex) { if err != nil { - t.Fatalf("test %d #5, err: %v", i, err) + t.Fatalf("test %d #9, err: %v", i, err) } if decoded.Cmp(&intSample) != 0 { - t.Fatalf("test %d #6, got %v, exp %v", i, decoded, intSample) + t.Fatalf("test %d #10, got %v, exp %v", i, decoded, intSample) } } // z.SetFromHex err = decoded.SetFromHex(wantHex) { if err != nil { - t.Fatalf("test %d #5, err: %v", i, err) + t.Fatalf("test %d #11, err: %v", i, err) } if decoded.Cmp(&intSample) != 0 { - t.Fatalf("test %d #6, got %v, exp %v", i, decoded, intSample) + t.Fatalf("test %d #12, got %v, exp %v", i, decoded, intSample) } } // UnmarshalText decoded = new(Int) { if err := decoded.UnmarshalText([]byte(wantHex)); err != nil { - t.Fatalf("test %d #7, err: %v", i, err) + t.Fatalf("test %d #13, err: %v", i, err) } if decoded.Cmp(&intSample) != 0 { - t.Fatalf("test %d #8, got %v, exp %v", i, decoded, intSample) + t.Fatalf("test %d #14, got %v, exp %v", i, decoded, intSample) } } // FromDecimal decoded, err = FromDecimal(wantDec) { if err != nil { - t.Fatalf("test %d #9, err: %v", i, err) + t.Fatalf("test %d #15, err: %v", i, err) } if decoded.Cmp(&intSample) != 0 { - t.Fatalf("test %d #10, got %v, exp %v", i, decoded, intSample) + t.Fatalf("test %d #16, got %v, exp %v", i, decoded, intSample) } } // Scan w string err = decoded.Scan(wantDec) { if err != nil { - t.Fatalf("test %d #9, err: %v", i, err) + t.Fatalf("test %d #17, err: %v", i, err) } if decoded.Cmp(&intSample) != 0 { - t.Fatalf("test %d #10, got %v, exp %v", i, decoded, intSample) + t.Fatalf("test %d #18, got %v, exp %v", i, decoded, intSample) } } // Scan w byte slice err = decoded.Scan([]byte(wantDec)) { if err != nil { - t.Fatalf("test %d #9, err: %v", i, err) + t.Fatalf("test %d #19, err: %v", i, err) } if decoded.Cmp(&intSample) != 0 { - t.Fatalf("test %d #10, got %v, exp %v", i, decoded, intSample) + t.Fatalf("test %d #20, got %v, exp %v", i, decoded, intSample) } } // Scan with neither string nor byte err = decoded.Scan(5) { if err == nil { - t.Fatalf("test %d #11, want error", i) + t.Fatalf("test %d #21, want error", i) } } } @@ -1318,3 +1320,154 @@ func TestNil(t *testing.T) { t.Fatal("want zero") } } + +func TestDecimal(t *testing.T) { + for i := uint(0); i < 255; i++ { + a := NewInt(1) + a.Lsh(a, i) + want := a.ToBig().Text(10) + if have := a.Dec(); have != want { + t.Errorf("want '%v' have '%v', \n", want, have) + } + // Op must not modify the original + if have := a.Dec(); have != want { + t.Errorf("want '%v' have '%v', \n", want, have) + } + } + // test zero-case + if have, want := new(Int).Dec(), new(big.Int).Text(10); have != want { + t.Errorf("have '%v', want '%v'", have, want) + } + { // max + max, _ := FromHex("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + maxb, _ := new(big.Int).SetString("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 0) + if have, want := max.Dec(), maxb.Text(10); have != want { + t.Errorf("have '%v', want '%v'", have, want) + } + } + { + max, _ := FromDecimal("29999999999999999999") + maxb, _ := new(big.Int).SetString("29999999999999999999", 0) + if have, want := max.Dec(), maxb.Text(10); have != want { + t.Errorf("have '%v', want '%v'", have, want) + } + } +} + +// prettyFmtBigInt formats n with thousand separators. +func prettyFmtBigInt(n *big.Int) string { + var ( + text = n.String() + buf = make([]byte, len(text)+len(text)/3) + comma = 0 + i = len(buf) - 1 + ) + for j := len(text) - 1; j >= 0; j, i = j-1, i-1 { + c := text[j] + + switch { + case c == '-': + buf[i] = c + case comma == 3: + buf[i] = ',' + i-- + comma = 0 + fallthrough + default: + buf[i] = c + comma++ + } + } + return string(buf[i+1:]) +} + +func TestPrettyDecimal(t *testing.T) { + for i := uint(0); i < 255; i++ { + a := NewInt(1) + a.Lsh(a, i) + want := prettyFmtBigInt(a.ToBig()) + if have := a.PrettyDec(','); have != want { + t.Errorf("%d: have '%v', want '%v'", i, have, want) + } + // Op must not modify the original + if have := a.PrettyDec(','); have != want { + t.Errorf("%d: have '%v', want '%v'", i, have, want) + } + } + // test zero-case + if have, want := new(Int).PrettyDec(','), prettyFmtBigInt(new(big.Int)); have != want { + t.Errorf("have '%v', want '%v'", have, want) + } +} + +func FuzzDecimal(f *testing.F) { + f.Fuzz(func(t *testing.T, aa, bb, cc, dd uint64) { + a := &Int{aa, bb, cc, dd} + { // Test Dec() + want := a.ToBig().Text(10) + if have := a.Dec(); have != want { + t.Errorf("want '%v' have '%v', \n", want, have) + } + // Op must not modify the original + if have := a.Dec(); have != want { + t.Errorf("want '%v' have '%v', \n", want, have) + } + } + { // Test PrettyDec + want := prettyFmtBigInt(a.ToBig()) + if have := a.PrettyDec(','); have != want { + t.Errorf("have '%v', want '%v'", have, want) + } + // Op must not modify the original + if have := a.PrettyDec(','); have != want { + t.Errorf("have '%v', want '%v'", have, want) + } + } + { // Test Hex() + want := fmt.Sprintf("%#x", a.ToBig()) + if have := a.Hex(); have != want { + t.Errorf("want '%v' have '%v', \n", want, have) + } + // Op must not modify the original + if have := a.Hex(); have != want { + t.Errorf("want '%v' have '%v', \n", want, have) + } + } + }) +} + +func BenchmarkDecimal(b *testing.B) { + var u256Ints []*Int + var bigints []*big.Int + + for i := uint(0); i < 255; i++ { + a := NewInt(1) + a.Lsh(a, i) + u256Ints = append(u256Ints, a) + bigints = append(bigints, a.ToBig()) + } + b.Run("ToDecimal/uint256", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + for _, z := range u256Ints { + _ = z.Dec() + } + } + }) + b.Run("ToPrettyDecimal/uint256", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + for _, z := range u256Ints { + _ = z.PrettyDec(',') + } + } + }) + b.Run("ToDecimal/big", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + for _, z := range bigints { + _ = z.Text(10) + } + } + }) +} diff --git a/decimal.go b/decimal.go index 3390a08d..7920e685 100644 --- a/decimal.go +++ b/decimal.go @@ -11,8 +11,103 @@ import ( const twoPow256Sub1 = "115792089237316195423570985008687907853269984665640564039457584007913129639935" +// Dec returns the decimal representation of z. func (z *Int) Dec() string { - return z.ToBig().String() + if z.IsZero() { + return "0" + } + if z.IsUint64() { + return strconv.FormatUint(z.Uint64(), 10) + } + // The max uint64 value being 18446744073709551615, the largest + // power-of-ten below that is 10000000000000000000. + // When we do a DivMod using that number, the remainder that we + // get back is the lower part of the output. + // + // The ascii-output of remainder will never exceed 19 bytes (since it will be + // below 10000000000000000000). + // + // Algorithm example using 100 as divisor + // + // 12345 % 100 = 45 (rem) + // 12345 / 100 = 123 (quo) + // -> output '45', continue iterate on 123 + var ( + // out is 98 bytes long: 78 (max size of a string without leading zeroes, + // plus slack so we can copy 19 bytes every iteration). + // We init it with zeroes, because when strconv appends the ascii representations, + // it will omit leading zeroes. + out = []byte("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") + divisor = NewInt(10000000000000000000) // 20 digits + y = new(Int).Set(z) // copy to avoid modifying z + pos = len(out) // position to write to + buf = make([]byte, 0, 19) // buffer to write uint64:s to + ) + for { + // Obtain Q and R for divisor + var quot Int + rem := udivrem(quot[:], y[:], divisor) + y.Set(") // Set Q for next loop + // Convert the R to ascii representation + buf = strconv.AppendUint(buf[:0], rem.Uint64(), 10) + // Copy in the ascii digits + copy(out[pos-len(buf):], buf) + if y.IsZero() { + break + } + // Move 19 digits left + pos -= 19 + } + // skip leading zeroes by only using the 'used size' of buf + return string(out[pos-len(buf):]) +} + +// PrettyDec returns the decimal representation of z, with thousands-separators. +func (z *Int) PrettyDec(separator byte) string { + if z.IsZero() { + return "0" + } + // See algorithm-description in Dec() + // This just also inserts comma while copying byte-for-byte instead + // of using copy(). + var ( + out = []byte("0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000") + divisor = NewInt(10000000000000000000) + y = new(Int).Set(z) // copy to avoid modifying z + pos = len(out) - 1 // position to write to + buf = make([]byte, 0, 19) // buffer to write uint64:s to + comma = 0 + ) + for { + var quot Int + rem := udivrem(quot[:], y[:], divisor) + y.Set(") // Set Q for next loop + buf = strconv.AppendUint(buf[:0], rem.Uint64(), 10) + for j := len(buf) - 1; j >= 0; j-- { + if comma == 3 { + out[pos] = separator + pos-- + comma = 0 + } + out[pos] = buf[j] + comma++ + pos-- + } + if y.IsZero() { + break + } + // Need to do zero-padding if we have more iterations coming + for j := 0; j < 19-len(buf); j++ { + if comma == 3 { + out[pos] = separator + pos-- + comma = 0 + } + comma++ + pos-- + } + } + return string(out[pos+1:]) } // FromDecimal is a convenience-constructor to create an Int from a