diff --git a/circle.yml b/circle.yml index effc4cbd..7a4d20de 100644 --- a/circle.yml +++ b/circle.yml @@ -19,14 +19,14 @@ commands: jobs: - go117: + go119: docker: - - image: cimg/go:1.17 + - image: cimg/go:1.19 steps: - run: name: "Install tools" command: | - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.42.0 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.50.1 - checkout - run: name: "Lint" @@ -40,18 +40,15 @@ jobs: command: bash <(curl -s https://codecov.io/bash) - restore_cache: keys: - - corpus-v2 + - corpus-v3 - run: name: "Fuzzing" command: | - go get -u github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build - go-fuzz-build - timeout --preserve-status --signal INT 1m go-fuzz -procs=2 - test ! "$(ls crashers)" + GOCACHE=/home/circleci/project/corpus-v3 go test . -run - -fuzz . -fuzztime 1m - save_cache: - key: corpus-v2-{{ epoch }} + key: corpus-v3-{{ epoch }} paths: - - corpus + - corpus-v3 - run: name: "Benchmark" command: go test -run=- -bench=. -benchmem @@ -78,40 +75,13 @@ jobs: name: "Test (PPC64 emulation)" command: qemu-ppc64-static uint256.test.ppc64 -test.v - go116: + go118: docker: - - image: cimg/go:1.16 + - image: cimg/go:1.18 steps: - checkout - test - go115: - docker: - - image: cimg/go:1.15 - steps: - - checkout - - test - - go114: - docker: - - image: cimg/go:1.14 - steps: - - checkout - - test - - go113: - docker: - - image: cimg/go:1.13 - steps: - - checkout - - test - - go112: - docker: - - image: cimg/go:1.12 - steps: - - checkout - - test @@ -119,12 +89,8 @@ workflows: version: 2 uint256: jobs: - - go117 - - go116 - - go115 - - go114 - - go113 - - go112 + - go119 + - go118 - bigendian: requires: - - go117 + - go119 diff --git a/conversion.go b/conversion.go index f1530f1e..6a234411 100644 --- a/conversion.go +++ b/conversion.go @@ -5,12 +5,17 @@ package uint256 import ( + "database/sql" + "database/sql/driver" + "encoding" "encoding/binary" + "encoding/json" "errors" "fmt" "io" "math/big" "math/bits" + "strings" ) const ( @@ -24,6 +29,16 @@ const ( _ uint = -(maxWords & ^(4 | 8)) // maxWords is 4 or 8. ) +// Compile time interface checks +var ( + _ driver.Valuer = (*Int)(nil) + _ sql.Scanner = (*Int)(nil) + _ encoding.TextMarshaler = (*Int)(nil) + _ encoding.TextUnmarshaler = (*Int)(nil) + _ json.Marshaler = (*Int)(nil) + _ json.Unmarshaler = (*Int)(nil) +) + // ToBig returns a big.Int version of z. func (z *Int) ToBig() *big.Int { b := new(big.Int) @@ -51,12 +66,24 @@ func FromBig(b *big.Int) (*Int, bool) { return z, overflow } +// SetFromHex sets z from the given string, interpreted as a hexadecimal number. +// OBS! This method is _not_ strictly identical to the (*big.Int).SetString(..., 16) method. +// Notable differences: +// - This method _require_ "0x" or "0X" prefix. +// - This method does not accept zero-prefixed hex, e.g. "0x0001" +// - This method does not accept underscore input, e.g. "100_000", +// - This method does not accept negative zero as valid, e.g "-0x0", +// - (this method does not accept any negative input as valid) +func (z *Int) SetFromHex(hex string) error { + z.Clear() + return z.fromHex(hex) +} + // fromHex is the internal implementation of parsing a hex-string. func (z *Int) fromHex(hex string) error { if err := checkNumberS(hex); err != nil { return err } - if len(hex) > 66 { return ErrBig256Range } @@ -92,6 +119,7 @@ func FromHex(hex string) (*Int, error) { // UnmarshalText implements encoding.TextUnmarshaler func (z *Int) UnmarshalText(input []byte) error { + z.Clear() return z.fromHex(string(input)) } @@ -147,7 +175,6 @@ func (z *Int) SetFromBig(b *big.Int) bool { // specification of minimum digits precision, output field // width, space or zero padding, and '-' for left or right // justification. -// func (z *Int) Format(s fmt.State, ch rune) { z.ToBig().Format(s, ch) } @@ -470,6 +497,11 @@ func (z *Int) MarshalText() ([]byte, error) { return []byte(z.Hex()), nil } +// MarshalJSON implements json.Marshaler. +func (z *Int) MarshalJSON() ([]byte, error) { + return []byte(`"` + z.Hex() + `"`), nil +} + // UnmarshalJSON implements json.Unmarshaler. func (z *Int) UnmarshalJSON(input []byte) error { if len(input) < 2 || input[0] != '"' || input[len(input)-1] != '"' { @@ -525,6 +557,53 @@ func (z *Int) Hex() string { return string(output[64-nibbles:]) } +// Scan implements the database/sql Scanner interface. +// It decodes a string, because that is what postgres uses for its numeric type +func (dst *Int) Scan(src interface{}) error { + if src == nil { + dst.Clear() + return nil + } + switch src := src.(type) { + case string: + splt := strings.SplitN(src, "e", 2) + if len(splt) == 1 { + return dst.SetFromDecimal(src) + } + if err := dst.SetFromDecimal(splt[0]); err != nil { + return err + } + if splt[1] == "0" { + return nil + } + exp := new(Int) + if err := exp.SetFromDecimal(splt[1]); err != nil { + return err + } + if !exp.IsUint64() || exp.Uint64() > uint64(len(twoPow256Sub1)) { + return ErrBig256Range + } + exp.Exp(NewInt(10), exp) + _, overflow := dst.MulOverflow(dst, exp) + if overflow { + return ErrBig256Range + } + return nil + case []byte: + return dst.SetFromDecimal(string(src)) + } + return fmt.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +// It encodes a base 10 string. +// In Postgres, this will work with both integer and the Numeric/Decimal types +// In MariaDB/MySQL, this will work with the Numeric/Decimal types up to 65 digits, however any more and you should use either VarChar or Char(79) +// In SqLite, use TEXT +func (src *Int) Value() (driver.Value, error) { + return src.ToBig().String(), nil +} + var ( ErrEmptyString = errors.New("empty hex string") ErrSyntax = errors.New("invalid hex string") diff --git a/conversion_test.go b/conversion_test.go index 7e7c82eb..b8e35afd 100644 --- a/conversion_test.go +++ b/conversion_test.go @@ -75,6 +75,65 @@ func TestFromBig(t *testing.T) { } } +func TestScanScientific(t *testing.T) { + intsub1 := new(Int) + _ = intsub1.fromDecimal(twoPow256Sub1) + cases := []struct { + in string + exp *Int + err string + }{ + { + in: "14e30", + exp: new(Int).Mul(NewInt(14), new(Int).Exp(NewInt(10), NewInt(30))), + }, + { + in: "1455522523e31", + exp: new(Int).Mul(NewInt(1455522523), new(Int).Exp(NewInt(10), NewInt(31))), + }, + { + in: twoPow256Sub1 + "e0", + exp: intsub1, + }, + { + in: "1e25352", + err: ErrBig256Range.Error(), + }, + { + in: "1213128763127863781263781263781263781263781263871263871268371268371263781627836128736128736127836127836127863781e0", + err: ErrBig256Range.Error(), + }, + { + in: twoPow256Sub1 + "e1", + err: ErrBig256Range.Error(), + }, + { + in: "1e253e52", + err: `strconv.ParseUint: parsing "253e52": invalid syntax`, + }, + { + in: "1e00000000000000000", + exp: NewInt(1), + }, + } + for tc, v := range cases { + have := "" + i := new(Int) + if err := i.Scan(v.in); err != nil { + have = err.Error() + } + if want := v.err; have != want { + t.Fatalf("test %d: wrong error, have '%s', want '%s'", tc, have, want) + } + if len(v.err) > 0 { + continue + } + if !v.exp.Eq(i) { + t.Fatalf("test %d: got %#x exp %#x", tc, i, v.exp) + } + } +} + func TestFromBigOverflow(t *testing.T) { _, o := FromBig(new(big.Int).SetBytes(hex2Bytes("ababee444444444444ffcc333333333333ddaa222222222222bb8811111111111199"))) if !o { @@ -720,16 +779,20 @@ func TestEnDecode(t *testing.T) { } var testSample = func(i int, bigSample big.Int, intSample Int) { // Encoding - exp := fmt.Sprintf("0x%s", bigSample.Text(16)) + wantHex := fmt.Sprintf("0x%s", bigSample.Text(16)) + wantDec := bigSample.Text(10) - if got := intSample.Hex(); exp != got { - t.Fatalf("test %d #1, got %v, exp %v", i, got, exp) + if got := intSample.Hex(); wantHex != got { + t.Fatalf("test %d #1, got %v, exp %v", i, got, wantHex) + } + if got := intSample.String(); wantHex != got { + t.Fatalf("test %d #2, got %v, exp %v", i, got, wantHex) } - if got := intSample.String(); exp != got { - t.Fatalf("test %d #2, got %v, exp %v", i, got, exp) + if got, _ := intSample.MarshalText(); wantHex != string(got) { + t.Fatalf("test %d #3, got %v, exp %v", i, got, wantHex) } - if got, _ := intSample.MarshalText(); exp != string(got) { - t.Fatalf("test %d #3, got %v, exp %v", i, got, exp) + if got, _ := intSample.Value(); wantDec != got.(string) { + t.Fatalf("test %d #4, got %v, exp %v", i, got, wantHex) } { // Json jsonEncoded, err := json.Marshal(&jsonStruct{&intSample}) @@ -746,21 +809,74 @@ func TestEnDecode(t *testing.T) { } } // Decoding - dec, err := FromHex(exp) - if err != nil { - t.Fatalf("test %d #5, err: %v", i, err) + // + // FromHex + decoded, err := FromHex(wantHex) + { + if err != nil { + t.Fatalf("test %d #5, err: %v", i, err) + } + if decoded.Cmp(&intSample) != 0 { + t.Fatalf("test %d #6, got %v, exp %v", i, decoded, intSample) + } } - if dec.Cmp(&intSample) != 0 { - t.Fatalf("test %d #6, got %v, exp %v", i, dec, intSample) + // z.SetFromHex + err = decoded.SetFromHex(wantHex) + { + if err != nil { + t.Fatalf("test %d #5, err: %v", i, err) + } + if decoded.Cmp(&intSample) != 0 { + t.Fatalf("test %d #6, got %v, exp %v", i, decoded, intSample) + } } - dec = new(Int) - if err := dec.UnmarshalText([]byte(exp)); err != nil { - t.Fatalf("test %d #7, err: %v", i, err) + // UnmarshalText + decoded = new(Int) + { + if err := decoded.UnmarshalText([]byte(wantHex)); err != nil { + t.Fatalf("test %d #7, err: %v", i, err) + } + if decoded.Cmp(&intSample) != 0 { + t.Fatalf("test %d #8, got %v, exp %v", i, decoded, intSample) + } } - if dec.Cmp(&intSample) != 0 { - t.Fatalf("test %d #8, got %v, exp %v", i, dec, intSample) + // FromDecimal + decoded, err = FromDecimal(wantDec) + { + if err != nil { + t.Fatalf("test %d #9, err: %v", i, err) + } + if decoded.Cmp(&intSample) != 0 { + t.Fatalf("test %d #10, got %v, exp %v", i, decoded, intSample) + } + } + // Scan w string + err = decoded.Scan(wantDec) + { + if err != nil { + t.Fatalf("test %d #9, err: %v", i, err) + } + if decoded.Cmp(&intSample) != 0 { + t.Fatalf("test %d #10, got %v, exp %v", i, decoded, intSample) + } + } + // Scan w byte slice + err = decoded.Scan([]byte(wantDec)) + { + if err != nil { + t.Fatalf("test %d #9, err: %v", i, err) + } + if decoded.Cmp(&intSample) != 0 { + t.Fatalf("test %d #10, got %v, exp %v", i, decoded, intSample) + } + } + // Scan with neither string nor byte + err = decoded.Scan(5) + { + if err == nil { + t.Fatalf("test %d #11, want error", i) + } } - } for i, bigSample := range big256Samples { intSample := int256Samples[i] @@ -772,3 +888,13 @@ func TestEnDecode(t *testing.T) { testSample(i, bigSample, intSample) } } + +func TestNil(t *testing.T) { + a := NewInt(1337) + if err := a.Scan(nil); err != nil { + t.Fatal(err) + } + if !a.IsZero() { + t.Fatal("want zero") + } +} diff --git a/decimal.go b/decimal.go new file mode 100644 index 00000000..3390a08d --- /dev/null +++ b/decimal.go @@ -0,0 +1,117 @@ +// uint256: Fixed size 256-bit math library +// Copyright 2020 uint256 Authors +// SPDX-License-Identifier: BSD-3-Clause + +package uint256 + +import ( + "io" + "strconv" +) + +const twoPow256Sub1 = "115792089237316195423570985008687907853269984665640564039457584007913129639935" + +func (z *Int) Dec() string { + return z.ToBig().String() +} + +// FromDecimal is a convenience-constructor to create an Int from a +// decimal (base 10) string. Numbers larger than 256 bits are not accepted. +func FromDecimal(decimal string) (*Int, error) { + var z Int + if err := z.SetFromDecimal(decimal); err != nil { + return nil, err + } + return &z, nil +} + +// SetFromDecimal sets z from the given string, interpreted as a decimal number. +// OBS! This method is _not_ strictly identical to the (*big.Int).SetString(..., 10) method. +// Notable differences: +// - This method does not accept underscore input, e.g. "100_000", +// - This method does not accept negative zero as valid, e.g "-0", +// - (this method does not accept any negative input as valid)) +func (z *Int) SetFromDecimal(s string) (err error) { + // Remove max one leading + + if len(s) > 0 && s[0] == '+' { + s = s[1:] + } + // Remove any number of leading zeroes + if len(s) > 0 && s[0] == '0' { + var i int + var c rune + for i, c = range s { + if c != '0' { + break + } + } + s = s[i:] + } + if len(s) < len(twoPow256Sub1) { + return z.fromDecimal(s) + } + if len(s) == len(twoPow256Sub1) { + if s > twoPow256Sub1 { + return ErrBig256Range + } + return z.fromDecimal(s) + } + return ErrBig256Range +} + +// multipliers holds the values that are needed for fromDecimal +var multipliers = [5]*Int{ + nil, // represents first round, no multiplication needed + {10000000000000000000, 0, 0, 0}, // 10 ^ 19 + {687399551400673280, 5421010862427522170, 0, 0}, // 10 ^ 38 + {5332261958806667264, 17004971331911604867, 2938735877055718769, 0}, // 10 ^ 57 + {0, 8607968719199866880, 532749306367912313, 1593091911132452277}, // 10 ^ 76 +} + +// fromDecimal is a helper function to only ever be called via SetFromDecimal +// this function takes a string and chunks it up, calling ParseUint on it up to 5 times +// these chunks are then multiplied by the proper power of 10, then added together. +func (z *Int) fromDecimal(bs string) error { + // first clear the input + z.Clear() + // the maximum value of uint64 is 18446744073709551615, which is 20 characters + // one less means that a string of 19 9's is always within the uint64 limit + var ( + num uint64 + err error + remaining = len(bs) + ) + if remaining == 0 { + return io.EOF + } + // We proceed in steps of 19 characters (nibbles), from least significant to most significant. + // This means that the first (up to) 19 characters do not need to be multiplied. + // In the second iteration, our slice of 19 characters needs to be multipleied + // by a factor of 10^19. Et cetera. + for i, mult := range multipliers { + if remaining <= 0 { + return nil // Done + } else if remaining > 19 { + num, err = strconv.ParseUint(bs[remaining-19:remaining], 10, 64) + } else { + // Final round + num, err = strconv.ParseUint(bs, 10, 64) + } + if err != nil { + return err + } + // add that number to our running total + if i == 0 { + z.SetUint64(num) + } else { + base := NewInt(num) + z.Add(z, base.Mul(base, mult)) + } + // Chop off another 19 characters + if remaining > 19 { + bs = bs[0 : remaining-19] + } + remaining -= 19 + } + return nil +} diff --git a/decimal_test.go b/decimal_test.go new file mode 100644 index 00000000..be106689 --- /dev/null +++ b/decimal_test.go @@ -0,0 +1,181 @@ +// uint256: Fixed size 256-bit math library +// Copyright 2020 uint256 Authors +// SPDX-License-Identifier: BSD-3-Clause + +package uint256 + +import ( + "fmt" + "math/big" + "testing" +) + +// Test SetFromDecimal +func testSetFromDec(tc string) error { + a := new(Int).SetAllOne() + err := a.SetFromDecimal(tc) + { // Check the FromDecimal too + b, err2 := FromDecimal(tc) + if (err == nil) != (err2 == nil) { + return fmt.Errorf("err != err2: %v %v", err, err2) + } + if err == nil { + if a.Cmp(b) != 0 { + return fmt.Errorf("a != b: %v %v", a, b) + } + } + } + // If input is negative, we should eror + if len(tc) > 0 && tc[0] == '-' { + if err == nil { + return fmt.Errorf("want error on negative input") + } + return nil + } + // Need to compare with big.Int + bigA, ok := big.NewInt(0).SetString(tc, 10) + if !ok { + if err == nil { + return fmt.Errorf("want error") + } + return nil // both agree that input is bad + } + if bigA.BitLen() > 256 { + if err == nil { + return fmt.Errorf("want error (bitlen > 256)") + } + return nil + } + want := bigA.String() + have := a.Dec() + if want != have { + return fmt.Errorf("want %v, have %v", want, have) + } + return nil +} + +var cases = []string{ + "0000000000000000000000000000000000000000000000000000000000000000000000000000000", + "0000000000000000000000000000000000000000000000000000000000000000000000000000097", + "-000000000000000000000000000000000000000000000000000000000000000000000000000000", + "1157920892373161954235709850086879078532699846656405640394575840079131296399351", + "215792089237316195423570985008687907853269984665640564039457584007913129639935", + "115792089237316195423570985008687907853269984665640564039457584007913129639935", + "15792089237316195423570985008687907853269984665640564039457584007913129639935", + "+115792089237316195423570985008687907853269984665640564039457584007913129639935", + "115792089237316195423570985008687907853269984665640564039457584007913129639936", + "115792089237316195423570985008687907853269984665640564039457584007913129639935", + "+0b00000000000000000000000000000000000000000000000000000000000000010", + "340282366920938463463374607431768211456", + "3402823669209384634633746074317682114561", + "+3402823669209384634633746074317682114561", + "+-3402823669209384634633746074317682114561", + "40282366920938463463374607431768211456", + "00000000000000000000000097", + "184467440737095516161", + "8446744073709551616", + "banana", + "+0x10", + "000", + "+000", + "010", + "0xab", + "-10", + "01", + "ab", + "0", + "-0", + "+0", + "", + "熊熊熊熊熊熊熊熊", + "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "-0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "0x10101011010", + "熊熊熊熊熊熊熊熊", + "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "-0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "-0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "0x10000000000000000000000000000000000000000000000000000000000000000", + "+0x10000000000000000000000000000000000000000000000000000000000000000", + "+0x00000000000000000000000000000000000000000000000000000000000000000", + "-0x00000000000000000000000000000000000000000000000000000000000000000", +} + +func TestStringScan(t *testing.T) { + for i, tc := range cases { + if err := testSetFromDec(tc); err != nil { + t.Errorf("test %d, input '%s', SetFromDecimal err: %v", i, tc, err) + } + // TODO testSetFromHex(tc) + } +} + +func FuzzBase10StringCompare(f *testing.F) { + for _, tc := range cases { + f.Add(tc) + } + f.Fuzz(func(t *testing.T, tc string) { + if err := testSetFromDec(tc); err != nil { + t.Errorf("input '%s', SetFromDecimal err: %v", tc, err) + } + // TODO testSetFromHex(tc) + }) +} + +func BenchmarkFromDecimalString(b *testing.B) { + input := twoPow256Sub1 + + b.Run("bigint", func(b *testing.B) { + val := new(big.Int) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + for j := 1; j < len(input); j++ { + if _, ok := val.SetString(input[:j], 10); !ok { + b.Fatalf("Error on %v", string(input[:j])) + } + } + } + }) + + b.Run("u256", func(b *testing.B) { + val := new(Int) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + for j := 1; j < len(input); j++ { + if err := val.SetFromDecimal(input[:j]); err != nil { + b.Fatalf("%v: %v", err, string(input[:j])) + } + } + } + }) +} + +func BenchmarkFromHexString(b *testing.B) { + input := "0xf131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303" + b.Run("bigint", func(b *testing.B) { + val := new(big.Int) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + for j := 3; j < len(input); j++ { + if _, ok := val.SetString(input[:j], 0); !ok { + b.Fatalf("Error on %v", string(input[:j])) + } + } + } + }) + b.Run("u256", func(b *testing.B) { + val := new(Int) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + for j := 3; j < len(input); j++ { + if err := val.SetFromHex(input[:j]); err != nil { + b.Fatalf("%v: %v", err, string(input[:j])) + } + } + } + }) +} diff --git a/fuzz.go b/fuzz.go index 790d6bf1..e085644d 100644 --- a/fuzz.go +++ b/fuzz.go @@ -304,3 +304,50 @@ func fuzzTernaryOp(data []byte) int { } return 1 } + +// Test SetFromDecimal +func testSetFromDecForFuzzing(tc string) error { + a := new(Int).SetAllOne() + err := a.SetFromDecimal(tc) + // If input is negative, we should eror + if len(tc) > 0 && tc[0] == '-' { + if err == nil { + return fmt.Errorf("want error on negative input") + } + return nil + } + // Need to compare with big.Int + bigA, ok := big.NewInt(0).SetString(tc, 10) + if !ok { + if err == nil { + return fmt.Errorf("want error") + } + return nil // both agree that input is bad + } + if bigA.BitLen() > 256 { + if err == nil { + return fmt.Errorf("want error (bitlen > 256)") + } + return nil + } + want := bigA.String() + have := a.Dec() + if want != have { + return fmt.Errorf("want %v, have %v", want, have) + } + if _, err := a.Value(); err != nil { + return fmt.Errorf("fail to Value() %s, got err %s", tc, err) + } + return nil +} + +func FuzzSetString(data []byte) int { + if len(data) > 512 { + // Too large, makes no sense + return -1 + } + if err := testSetFromDecForFuzzing(string(data)); err != nil { + panic(err) + } + return 1 +} diff --git a/go.mod b/go.mod index a38ade78..26425ac6 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/holiman/uint256 -go 1.13 +go 1.18 diff --git a/oss-fuzz.sh b/oss-fuzz.sh index 7364dc78..ca516c08 100644 --- a/oss-fuzz.sh +++ b/oss-fuzz.sh @@ -1,2 +1,3 @@ #!/bin/bash -eu compile_go_fuzzer github.com/holiman/uint256 Fuzz uint256Fuzz +compile_go_fuzzer github.com/holiman/uint256 FuzzSetString uint256FuzzSetString