DO NOT MERGE: indexer 2.10.0

.circleci/config.yml

@@ -83,7 +83,7 @@ commands:
           name: Install python and other python dependencies
           command: |
             sudo apt update
-            sudo apt -y install python3 python3-pip python3-setuptools python3-wheel libboost-all-dev libffi-dev
+            sudo apt -y install python3 python3-pip python3-setuptools python3-wheel libboost-math-dev libffi-dev
             pip3 install -r misc/requirements.txt
 
       - run:

.version

@@ -1 +1 @@
-2.9.0
+2.10.0

Makefile

@@ -11,6 +11,8 @@ GOLDFLAGS += -X github.com/algorand/indexer/version.CompileTime=$(shell date -u
 GOLDFLAGS += -X github.com/algorand/indexer/version.GitDecorateBase64=$(shell git log -n 1 --pretty="%D"|base64|tr -d ' \n')
 GOLDFLAGS += -X github.com/algorand/indexer/version.ReleaseVersion=$(shell cat .version)
 
+COVERPKG := $(shell go list ./...  | grep -v '/cmd/' | egrep -v '(testing|test|mocks)$$' |  paste -s -d, - )
+
 # Used for e2e test
 export GO_IMAGE = golang:$(shell go version | cut -d ' ' -f 3 | tail -c +3 )
 
@@ -45,7 +47,7 @@ fakepackage: go-algorand
 	misc/release.py --host-only --outdir $(PKG_DIR) --fake-release
 
 test: idb/mocks/IndexerDb.go cmd/algorand-indexer/algorand-indexer
-	go test ./... -coverprofile=coverage.txt -covermode=atomic
+	go test -coverpkg=$(COVERPKG) ./... -coverprofile=coverage.txt -covermode=atomic
 
 lint: go-algorand
 	golint -set_exit_status ./...
@@ -75,7 +77,7 @@ test-generate:
 	test/test_generate.py
 
 nightly-setup:
-	cd third_party/go-algorand && git pull origin master
+	cd third_party/go-algorand && git fetch && git reset --hard origin/master
 
 nightly-teardown:
 	git submodule update

README.md

@@ -112,6 +112,50 @@ When `--token your-token` is provided, an authentication header is required. For
 ~$ curl localhost:8980/transactions -H "X-Indexer-API-Token: your-token"
 ```
 
+## Disabling Parameters 
+
+The Indexer has the ability to selectively enable or disable parameters for endpoints.  Disabling a "required" parameter will result in the entire endpoint being disabled while disabling an "optional" parameter will cause an error to be returned only if the parameter is provided.
+
+### Viewing the Current Configuration
+
+The Indexer has a default set of disabled parameters.  To view the disabled parameters issue:
+```
+~$ algorand-indexer api-config
+```
+
+This will output ONLY the disabled parameters in a YAML configuration.  To view all parameters (both enabled and disabled) issue:
+
+```
+~$ algorand-indexer api-config --all
+```
+
+### Interpreting The Configuration
+
+Below is a snippet of the output from `algorand-indexer api-config`:
+
+```
+/v2/accounts:
+    optional:
+        - currency-greater-than: disabled
+        - currency-less-than: disabled
+/v2/assets/{asset-id}/transactions:
+    optional:
+        - note-prefix: disabled
+        - tx-type: disabled
+        - sig-type: disabled
+        - before-time: disabled
+        - after-time: disabled
+        - currency-greater-than: disabled
+        - currency-less-than: disabled
+        - address-role: disabled
+        - exclude-close-to: disabled
+        - rekey-to: disabled
+    required:
+        - asset-id: disabled
+```
+
+Seeing this we know that the `/v2/accounts` endpoint will return an error if either `currency-greater-than` or `currency-less-than` is provided.  Additionally, because a "required" parameter is provided for `/v2/assets/{asset-id}/transactions` then we know this entire endpoint is disabled.  The optional parameters are provided so that you can understand what else is disabled if you enable all "required" parameters.
+
 ## Metrics
 
 The `/metrics` endpoint is configured with the `--metrics-mode` option and configures if and how [Prometheus](https://prometheus.io/) formatted metrics are generated.
@@ -134,20 +178,33 @@ If the maximum number of connections/active queries is reached, subsequent conne
 
 Settings can be provided from the command line, a configuration file, or an environment variable
 
-| Command Line Flag (long) | (short) | Config File                | Environment Variable               |
-| ------------------------ | ------- | -------------------------- | ---------------------------------- |
-| postgres                 | P       | postgres-connection-string | INDEXER_POSTGRES_CONNECTION_STRING |
-| pidfile                  |         | pidfile                    | INDEXER_PIDFILE                    |
-| algod                    | d       | algod-data-dir             | INDEXER_ALGOD_DATA_DIR             |
-| algod-net                |         | algod-address              | INDEXER_ALGOD_ADDRESS              |
-| algod-token              |         | algod-token                | INDEXER_ALGOD_TOKEN                |
-| genesis                  | g       | genesis                    | INDEXER_GENESIS                    |
-| server                   | S       | server-address             | INDEXER_SERVER_ADDRESS             |
-| no-algod                 |         | no-algod                   | INDEXER_NO_ALGOD                   |
-| token                    | t       | api-token                  | INDEXER_API_TOKEN                  |
-| dev-mode                 |         | dev-mode                   | INDEXER_DEV_MODE                   |
-| metrics-mode             |         | metrics-mode               | INDEXER_METRICS_MODE               |
-| max-conn                 |         | max-conn                   | INDEXER_MAX_CONN                   |
+| Command Line Flag (long)      | (short) | Config File                   | Environment Variable                  |
+|-------------------------------|---------|-------------------------------|---------------------------------------|
+| postgres                      | P       | postgres-connection-string    | INDEXER_POSTGRES_CONNECTION_STRING    |
+| pidfile                       |         | pidfile                       | INDEXER_PIDFILE                       |
+| algod                         | d       | algod-data-dir                | INDEXER_ALGOD_DATA_DIR                |
+| algod-net                     |         | algod-address                 | INDEXER_ALGOD_ADDRESS                 |
+| algod-token                   |         | algod-token                   | INDEXER_ALGOD_TOKEN                   |
+| genesis                       | g       | genesis                       | INDEXER_GENESIS                       |
+| server                        | S       | server-address                | INDEXER_SERVER_ADDRESS                |
+| no-algod                      |         | no-algod                      | INDEXER_NO_ALGOD                      |
+| token                         | t       | api-token                     | INDEXER_API_TOKEN                     |
+| dev-mode                      |         | dev-mode                      | INDEXER_DEV_MODE                      |
+| metrics-mode                  |         | metrics-mode                  | INDEXER_METRICS_MODE                  |
+| max-conn                      |         | max-conn                      | INDEXER_MAX_CONN                      |
+| write-timeout                 |         | write-timeout                 | INDEXER_WRITE_TIMEOUT                 |
+| read-timeout                  |         | read-timeout                  | INDEXER_READ_TIMEOUT                  |
+| max-api-resources-per-account |         | max-api-resources-per-account | INDEXER_MAX_API_RESOURCES_PER_ACCOUNT |
+| max-transactions-limit        |         | max-transactions-limit        | INDEXER_MAX_TRANSACTIONS_LIMIT        |
+| default-transactions-limit    |         | default-transactions-limit    | INDEXER_DEFAULT_TRANSACTIONS_LIMIT    |
+| max-accounts-limit            |         | max-accounts-limit            | INDEXER_MAX_ACCOUNTS_LIMIT            |
+| default-accounts-limit        |         | default-accounts-limit        | INDEXER_DEFAULT_ACCOUNTS_LIMIT        |
+| max-assets-limit              |         | max-assets-limit              | INDEXER_MAX_ASSETS_LIMIT              |
+| default-assets-limit          |         | default-assets-limit          | INDEXER_DEFAULT_ASSETS_LIMIT          |
+| max-balances-limit            |         | max-balances-limit            | INDEXER_MAX_BALANCES_LIMIT            |
+| default-balances-limit        |         | default-balances-limit        | INDEXER_DEFAULT_BALANCES_LIMIT        |
+| max-applications-limit        |         | max-applications-limit        | INDEXER_MAX_APPLICATIONS_LIMIT        |
+| default-applications-limit    |         | default-applications-limit    | INDEXER_DEFAULT_APPLICATIONS_LIMIT    |
 
 ## Command line
 

accounting/eval_preload.go

@@ -0,0 +1,202 @@
+package accounting
+
+import (
+	"github.com/algorand/go-algorand/data/basics"
+	"github.com/algorand/go-algorand/data/transactions"
+	"github.com/algorand/go-algorand/ledger"
+	"github.com/algorand/go-algorand/protocol"
+)
+
+// Add requests for asset and app creators to `assetsReq` and `appsReq` for the given
+// transaction.
+func addToCreatorsRequest(stxnad *transactions.SignedTxnWithAD, assetsReq map[basics.AssetIndex]struct{}, appsReq map[basics.AppIndex]struct{}) {
+	txn := &stxnad.Txn
+
+	switch txn.Type {
+	case protocol.AssetConfigTx:
+		fields := &txn.AssetConfigTxnFields
+		if fields.ConfigAsset != 0 {
+			assetsReq[fields.ConfigAsset] = struct{}{}
+		}
+	case protocol.AssetTransferTx:
+		fields := &txn.AssetTransferTxnFields
+		if fields.XferAsset != 0 {
+			assetsReq[fields.XferAsset] = struct{}{}
+		}
+	case protocol.AssetFreezeTx:
+		fields := &txn.AssetFreezeTxnFields
+		if fields.FreezeAsset != 0 {
+			assetsReq[fields.FreezeAsset] = struct{}{}
+		}
+	case protocol.ApplicationCallTx:
+		fields := &txn.ApplicationCallTxnFields
+		if fields.ApplicationID != 0 {
+			appsReq[fields.ApplicationID] = struct{}{}
+		}
+		for _, index := range fields.ForeignApps {
+			appsReq[index] = struct{}{}
+		}
+		for _, index := range fields.ForeignAssets {
+			assetsReq[index] = struct{}{}
+		}
+	}
+
+	for i := range stxnad.ApplyData.EvalDelta.InnerTxns {
+		addToCreatorsRequest(&stxnad.ApplyData.EvalDelta.InnerTxns[i], assetsReq, appsReq)
+	}
+}
+
+// MakePreloadCreatorsRequest makes a request for preloading creators in the batch mode.
+func MakePreloadCreatorsRequest(payset transactions.Payset) (map[basics.AssetIndex]struct{}, map[basics.AppIndex]struct{}) {
+	assetsReq := make(map[basics.AssetIndex]struct{}, len(payset))
+	appsReq := make(map[basics.AppIndex]struct{}, len(payset))
+
+	for i := range payset {
+		addToCreatorsRequest(&payset[i].SignedTxnWithAD, assetsReq, appsReq)
+	}
+
+	return assetsReq, appsReq
+}
+
+// Add requests for account data and account resources to `addressesReq` and
+// `resourcesReq` respectively for the given transaction.
+func addToAccountsResourcesRequest(stxnad *transactions.SignedTxnWithAD, assetCreators map[basics.AssetIndex]ledger.FoundAddress, appCreators map[basics.AppIndex]ledger.FoundAddress, addressesReq map[basics.Address]struct{}, resourcesReq map[basics.Address]map[ledger.Creatable]struct{}) {
+	setResourcesReq := func(addr basics.Address, creatable ledger.Creatable) {
+		c, ok := resourcesReq[addr]
+		if !ok {
+			c = make(map[ledger.Creatable]struct{})
+			resourcesReq[addr] = c
+		}
+		c[creatable] = struct{}{}
+	}
+
+	txn := &stxnad.Txn
+
+	addressesReq[txn.Sender] = struct{}{}
+
+	switch txn.Type {
+	case protocol.PaymentTx:
+		fields := &txn.PaymentTxnFields
+		addressesReq[fields.Receiver] = struct{}{}
+		// Close address is optional.
+		if !fields.CloseRemainderTo.IsZero() {
+			addressesReq[fields.CloseRemainderTo] = struct{}{}
+		}
+	case protocol.AssetConfigTx:
+		fields := &txn.AssetConfigTxnFields
+		if fields.ConfigAsset == 0 {
+			if stxnad.ApplyData.ConfigAsset != 0 {
+				creatable := ledger.Creatable{
+					Index: basics.CreatableIndex(stxnad.ApplyData.ConfigAsset),
+					Type:  basics.AssetCreatable,
+				}
+				setResourcesReq(txn.Sender, creatable)
+			}
+		} else {
+			if creator := assetCreators[fields.ConfigAsset]; creator.Exists {
+				creatable := ledger.Creatable{
+					Index: basics.CreatableIndex(fields.ConfigAsset),
+					Type:  basics.AssetCreatable,
+				}
+				addressesReq[creator.Address] = struct{}{}
+				setResourcesReq(creator.Address, creatable)
+			}
+		}
+	case protocol.AssetTransferTx:
+		fields := &txn.AssetTransferTxnFields
+		creatable := ledger.Creatable{
+			Index: basics.CreatableIndex(fields.XferAsset),
+			Type:  basics.AssetCreatable,
+		}
+		if creator := assetCreators[fields.XferAsset]; creator.Exists {
+			setResourcesReq(creator.Address, creatable)
+		}
+		source := txn.Sender
+		// If asset sender is non-zero, it is a clawback transaction. Otherwise,
+		// the transaction sender address is used.
+		if !fields.AssetSender.IsZero() {
+			source = fields.AssetSender
+		}
+		addressesReq[source] = struct{}{}
+		setResourcesReq(source, creatable)
+		addressesReq[fields.AssetReceiver] = struct{}{}
+		setResourcesReq(fields.AssetReceiver, creatable)
+		// Asset close address is optional.
+		if !fields.AssetCloseTo.IsZero() {
+			addressesReq[fields.AssetCloseTo] = struct{}{}
+			setResourcesReq(fields.AssetCloseTo, creatable)
+		}
+	case protocol.AssetFreezeTx:
+		fields := &txn.AssetFreezeTxnFields
+		creatable := ledger.Creatable{
+			Index: basics.CreatableIndex(fields.FreezeAsset),
+			Type:  basics.AssetCreatable,
+		}
+		if creator := assetCreators[fields.FreezeAsset]; creator.Exists {
+			setResourcesReq(creator.Address, creatable)
+		}
+		setResourcesReq(fields.FreezeAccount, creatable)
+	case protocol.ApplicationCallTx:
+		fields := &txn.ApplicationCallTxnFields
+		if fields.ApplicationID == 0 {
+			if stxnad.ApplyData.ApplicationID != 0 {
+				creatable := ledger.Creatable{
+					Index: basics.CreatableIndex(stxnad.ApplyData.ApplicationID),
+					Type:  basics.AppCreatable,
+				}
+				setResourcesReq(txn.Sender, creatable)
+			}
+		} else {
+			creatable := ledger.Creatable{
+				Index: basics.CreatableIndex(fields.ApplicationID),
+				Type:  basics.AppCreatable,
+			}
+			if creator := appCreators[fields.ApplicationID]; creator.Exists {
+				addressesReq[creator.Address] = struct{}{}
+				setResourcesReq(creator.Address, creatable)
+			}
+			setResourcesReq(txn.Sender, creatable)
+		}
+		for _, address := range fields.Accounts {
+			addressesReq[address] = struct{}{}
+		}
+		for _, index := range fields.ForeignApps {
+			if creator := appCreators[index]; creator.Exists {
+				creatable := ledger.Creatable{
+					Index: basics.CreatableIndex(index),
+					Type:  basics.AppCreatable,
+				}
+				setResourcesReq(creator.Address, creatable)
+			}
+		}
+		for _, index := range fields.ForeignAssets {
+			if creator := assetCreators[index]; creator.Exists {
+				creatable := ledger.Creatable{
+					Index: basics.CreatableIndex(index),
+					Type:  basics.AssetCreatable,
+				}
+				setResourcesReq(creator.Address, creatable)
+			}
+		}
+	}
+
+	for i := range stxnad.ApplyData.EvalDelta.InnerTxns {
+		addToAccountsResourcesRequest(
+			&stxnad.ApplyData.EvalDelta.InnerTxns[i], assetCreators, appCreators,
+			addressesReq, resourcesReq)
+	}
+}
+
+// MakePreloadAccountsResourcesRequest makes a request for preloading account data and
+// account resources in the batch mode.
+func MakePreloadAccountsResourcesRequest(payset transactions.Payset, assetCreators map[basics.AssetIndex]ledger.FoundAddress, appCreators map[basics.AppIndex]ledger.FoundAddress) (map[basics.Address]struct{}, map[basics.Address]map[ledger.Creatable]struct{}) {
+	addressesReq := make(map[basics.Address]struct{}, len(payset))
+	resourcesReq := make(map[basics.Address]map[ledger.Creatable]struct{}, len(payset))
+
+	for i := range payset {
+		addToAccountsResourcesRequest(
+			&payset[i].SignedTxnWithAD, assetCreators, appCreators, addressesReq, resourcesReq)
+	}
+
+	return addressesReq, resourcesReq
+}