Add support for v1.4.0 of the Rosetta API Specification

README.md

@@ -124,6 +124,7 @@ Flags:
                                                    reconciliation errors during development. (default true)
   -h, --help                                       help for check
       --inactive-reconciliation-concurrency uint   concurrency to use while fetching accounts during inactive reconciliation (default 4)
+      --inactive-reconciliation-frequency uint     the number of blocks to wait between inactive reconiliations on each account (default 250)
       --interesting-accounts string                Absolute path to a file listing all accounts to check on each block. Look
                                                    at the examples directory for an example of how to structure this file.
       --log-balance-changes                        log balance changes

cmd/check.go

@@ -136,6 +136,10 @@ of what one of these files looks like.`,
 	// while fetching accounts during inactive reconciliation.
 	InactiveReconciliationConcurrency uint64
 
+	// InactiveReconciliationFrequency is the number of blocks
+	// to wait between inactive reconiliations on each account.
+	InactiveReconciliationFrequency uint64
+
 	// LogBlocks determines if blocks are
 	// logged.
 	LogBlocks bool
@@ -215,6 +219,12 @@ func init() {
 		4,
 		"concurrency to use while fetching accounts during inactive reconciliation",
 	)
+	checkCmd.Flags().Uint64Var(
+		&InactiveReconciliationFrequency,
+		"inactive-reconciliation-frequency",
+		250,
+		"the number of blocks to wait between inactive reconiliations on each account",
+	)
 	checkCmd.Flags().BoolVar(
 		&LogBlocks,
 		"log-blocks",
@@ -585,6 +595,8 @@ func runCheckCmd(cmd *cobra.Command, args []string) {
 		reconciler.WithLookupBalanceByBlock(LookupBalanceByBlock),
 		reconciler.WithInterestingAccounts(interestingAccounts),
 		reconciler.WithSeenAccounts(seenAccounts),
+		reconciler.WithDebugLogging(LogReconciliations),
+		reconciler.WithInactiveFrequency(int64(InactiveReconciliationFrequency)),
 	)
 
 	syncerHandler := processor.NewSyncerHandler(

cmd/root.go

@@ -56,6 +56,6 @@ var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Print rosetta-cli version",
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("v0.2.5")
+		fmt.Println("v0.3.0")
 	},
 }

go.mod

@@ -4,7 +4,7 @@ go 1.13
 
 require (
 	github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
-	github.com/coinbase/rosetta-sdk-go v0.2.1-0.20200520212551-1f6ff35ffe98
+	github.com/coinbase/rosetta-sdk-go v0.3.1
 	github.com/dgraph-io/badger v1.6.1
 	github.com/fatih/color v1.9.0
 	github.com/golang/protobuf v1.4.2 // indirect

internal/processor/storage_helper.go

@@ -33,7 +33,7 @@ type BlockStorageHelper struct {
 
 	// Configuration settings
 	lookupBalanceByBlock bool
-	exemptAccounts       []*reconciler.AccountCurrency
+	exemptAccounts       map[string]struct{}
 }
 
 // NewBlockStorageHelper returns a new BlockStorageHelper.
@@ -43,11 +43,19 @@ func NewBlockStorageHelper(
 	lookupBalanceByBlock bool,
 	exemptAccounts []*reconciler.AccountCurrency,
 ) *BlockStorageHelper {
+	exemptMap := map[string]struct{}{}
+
+	// Pre-process exemptAccounts on initialization
+	// to provide fast lookup while syncing.
+	for _, account := range exemptAccounts {
+		exemptMap[types.Hash(account)] = struct{}{}
+	}
+
 	return &BlockStorageHelper{
 		network:              network,
 		fetcher:              fetcher,
 		lookupBalanceByBlock: lookupBalanceByBlock,
-		exemptAccounts:       exemptAccounts,
+		exemptAccounts:       exemptMap,
 	}
 }
 
@@ -97,12 +105,12 @@ func (h *BlockStorageHelper) Asserter() *asserter.Asserter {
 // ExemptFunc returns a parser.ExemptOperation.
 func (h *BlockStorageHelper) ExemptFunc() parser.ExemptOperation {
 	return func(op *types.Operation) bool {
-		return reconciler.ContainsAccountCurrency(
-			h.exemptAccounts,
-			&reconciler.AccountCurrency{
-				Account:  op.Account,
-				Currency: op.Amount.Currency,
-			},
-		)
+		thisAcct := types.Hash(&reconciler.AccountCurrency{
+			Account:  op.Account,
+			Currency: op.Amount.Currency,
+		})
+
+		_, exists := h.exemptAccounts[thisAcct]
+		return exists
 	}
 }

internal/processor/storage_helper_test.go

@@ -0,0 +1,94 @@
+// Copyright 2020 Coinbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package processor
+
+import (
+	"testing"
+
+	"github.com/coinbase/rosetta-sdk-go/reconciler"
+	"github.com/coinbase/rosetta-sdk-go/types"
+	"github.com/stretchr/testify/assert"
+)
+
+var (
+	opAmountCurrency = &reconciler.AccountCurrency{
+		Account: &types.AccountIdentifier{
+			Address: "hello",
+		},
+		Currency: &types.Currency{
+			Symbol:   "BTC",
+			Decimals: 8,
+		},
+	}
+)
+
+func TestExemptFunc(t *testing.T) {
+	var tests = map[string]struct {
+		exemptAccounts []*reconciler.AccountCurrency
+		exempt         bool
+	}{
+		"no exempt accounts": {},
+		"account not exempt": {
+			exemptAccounts: []*reconciler.AccountCurrency{
+				{
+					Account: &types.AccountIdentifier{
+						Address: "addr1",
+					},
+					Currency: opAmountCurrency.Currency,
+				},
+				{
+					Account: &types.AccountIdentifier{
+						Address: "addr2",
+					},
+					Currency: opAmountCurrency.Currency,
+				},
+			},
+		},
+		"account is exempt": {
+			exemptAccounts: []*reconciler.AccountCurrency{
+				{
+					Account: &types.AccountIdentifier{
+						Address: "addr1",
+					},
+					Currency: opAmountCurrency.Currency,
+				},
+				{
+					Account: &types.AccountIdentifier{
+						Address: "addr2",
+					},
+					Currency: opAmountCurrency.Currency,
+				},
+				opAmountCurrency,
+			},
+			exempt: true,
+		},
+	}
+
+	for name, test := range tests {
+		t.Run(name, func(t *testing.T) {
+			helper := NewBlockStorageHelper(nil, nil, false, test.exemptAccounts)
+
+			result := helper.ExemptFunc()(&types.Operation{
+				Account: opAmountCurrency.Account,
+				Amount: &types.Amount{
+					Value:    "100",
+					Currency: opAmountCurrency.Currency,
+				},
+			})
+
+			assert.Equal(t, test.exempt, result)
+		})
+	}
+}

internal/storage/block_storage_test.go

@@ -991,9 +991,17 @@ func (h *MockBlockStorageHelper) Asserter() *asserter.Asserter {
 
 func (h *MockBlockStorageHelper) ExemptFunc() parser.ExemptOperation {
 	return func(op *types.Operation) bool {
-		return reconciler.ContainsAccountCurrency(h.ExemptAccounts, &reconciler.AccountCurrency{
+		thisAcct := &reconciler.AccountCurrency{
 			Account:  op.Account,
 			Currency: op.Amount.Currency,
-		})
+		}
+
+		for _, acct := range h.ExemptAccounts {
+			if types.Hash(acct) == types.Hash(thisAcct) {
+				return true
+			}
+		}
+
+		return false
 	}
 }