Add handling of _minConfirmations reserved parameter (#1615)

* Drop requests with _minConfirmations larger than BLOCK_COUNT_HISTORY_LIMIT

* Move _minConfirmations request filtering upstream of API call

* Apply max minConfirmations by sponsor

.changeset/swift-kangaroos-cross.md

@@ -0,0 +1,8 @@
+---
+'@api3/airnode-adapter': minor
+'@api3/airnode-deployer': minor
+'@api3/airnode-node': minor
+'@api3/airnode-validator': minor
+---
+
+Add new \_minConfirmations reserved parameter to allow minConfirmations to be specified by a requester

packages/airnode-adapter/src/types.ts

@@ -62,7 +62,8 @@ export type BaseResponseType = typeof baseResponseTypes[number];
 // Use might pass a complex type (e.g. int256[3][]) which we cannot type
 export type ResponseType = string;
 
-// Reserved parameters specific for response processing i.e. excludes _gasPrice
+// Reserved parameters specific for response processing
+// i.e. excludes _gasPrice and _minConfirmations
 export interface ResponseReservedParameters {
   _path?: string;
   _times?: string;

packages/airnode-adapter/test/fixtures/ois.ts

@@ -78,6 +78,7 @@ export function buildOIS(overrides?: Partial<OIS>): OIS {
             default: '100000',
           },
           { name: '_gasPrice' },
+          { name: '_minConfirmations' },
         ],
         parameters: [
           {

packages/airnode-deployer/test/fixtures/config.aws.valid.json

@@ -206,6 +206,9 @@
             },
             {
               "name": "_gasPrice"
+            },
+            {
+              "name": "_minConfirmations"
             }
           ],
           "parameters": [

packages/airnode-deployer/test/fixtures/config.gcp.valid.json

@@ -206,6 +206,9 @@
             },
             {
               "name": "_gasPrice"
+            },
+            {
+              "name": "_minConfirmations"
             }
           ],
           "parameters": [

packages/airnode-node/src/adapters/http/parameters.test.ts

@@ -59,6 +59,7 @@ describe('getReservedParameters', () => {
         { name: '_path', default: 'prices.0.latest' },
         { name: '_times', default: '1000000' },
         { name: '_gasPrice' },
+        { name: '_minConfirmations' },
       ],
     };
   });
@@ -68,6 +69,12 @@ describe('getReservedParameters', () => {
       _type: 'bytes32',
       _path: 'updated.path',
     });
-    expect(res).toEqual({ _type: 'int256', _path: 'updated.path', _times: '1000000', _gasPrice: undefined });
+    expect(res).toEqual({
+      _type: 'int256',
+      _path: 'updated.path',
+      _times: '1000000',
+      _gasPrice: undefined,
+      _minConfirmations: undefined,
+    });
   });
 });

packages/airnode-node/src/api/index.test.ts

@@ -14,7 +14,14 @@ describe('callApi', () => {
     const spy = jest.spyOn(adapter, 'buildAndExecuteRequest') as any;
     spy.mockResolvedValueOnce({ data: { price: 1000 } });
     const requestedGasPrice = '100000000';
-    const parameters = { _type: 'int256', _path: 'price', from: 'ETH', _gasPrice: requestedGasPrice };
+    const requestedMinConfirmations = '0';
+    const parameters = {
+      _type: 'int256',
+      _path: 'price',
+      from: 'ETH',
+      _gasPrice: requestedGasPrice,
+      _minConfirmations: requestedMinConfirmations,
+    };
 
     const [logs, res] = await callApi({
       type: 'regular',
@@ -30,6 +37,7 @@ describe('callApi', () => {
         signature:
           '0xe92f5ee40ddb5aa42cab65fcdc025008b2bc026af80a7c93a9aac4e474f8a88f4f2bd861b9cf9a2b050bf0fd13e9714c4575cebbea658d7501e98c0963a5a38b1c',
       },
+      // _minConfirmations is processed before making API calls
       reservedParameterOverrides: {
         gasPrice: requestedGasPrice,
       },

packages/airnode-node/src/api/index.ts

@@ -226,6 +226,7 @@ export async function processSuccessfulApiCall(
   const { endpointName, oisTitle, parameters } = aggregatedApiCall;
   const ois = config.ois.find((o) => o.title === oisTitle)!;
   const endpoint = ois.endpoints.find((e) => e.name === endpointName)!;
+  // _minConfirmations is handled prior to the API call
   const { _type, _path, _times, _gasPrice } = getReservedParameters(endpoint, parameters);
 
   const goPostProcessApiSpecifications = await go(() => postProcessApiSpecifications(rawResponse.data, endpoint));

packages/airnode-node/src/coordinator/calls/disaggregation.test.ts

@@ -84,9 +84,7 @@ describe('disaggregate - Requests', () => {
     const stateWithResponses = coordinatorState.addResponses(state, { aggregatedApiCallsById, providerStates });
 
     const [logs, res] = disaggregation.disaggregate(stateWithResponses);
-    expect(logs).toEqual([
-      { level: 'ERROR', message: 'Unable to find matching aggregated API calls for Request:ethCall' },
-    ]);
+    expect(logs).toEqual([]);
     expect(res[0].requests.apiCalls.length).toEqual(0);
     expect((res[1].requests.apiCalls[0] as any as RegularApiCallSuccessResponse).data.encodedValue).toEqual('0x123');
     expect(res[1].requests.apiCalls[0].errorMessage).toEqual(undefined);

packages/airnode-node/src/coordinator/calls/disaggregation.ts

@@ -1,5 +1,4 @@
 import flatMap from 'lodash/flatMap';
-import { logger } from '@api3/airnode-utilities';
 import {
   RegularAggregatedApiCallsWithResponseById,
   ApiCall,
@@ -19,10 +18,9 @@ function updateApiCallResponses(
   const { logs, requests } = apiCalls.reduce(
     (acc, apiCall): UpdatedRequests<ApiCallWithResponse> => {
       const aggregatedApiCall = aggregatedApiCallsById[apiCall.id];
-      // There should always be a matching AggregatedApiCall. Something has gone wrong if there isn't
+      // An AggregatedApiCall may be dropped if there hasn't been minConfirmations block confirmations.
       if (!aggregatedApiCall) {
-        const log = logger.pend('ERROR', `Unable to find matching aggregated API calls for Request:${apiCall.id}`);
-        return { ...acc, logs: [...acc.logs, log] };
+        return acc;
       }
 
       // Add the error to the ApiCall

packages/airnode-node/src/evm/handlers/fetch-pending-requests.ts

@@ -15,6 +15,7 @@ export async function fetchPendingRequests(state: ProviderState<EVMProviderState
     blockHistoryLimit: state.settings.blockHistoryLimit,
     currentBlock: state.currentBlock!,
     minConfirmations: state.settings.minConfirmations,
+    mayOverrideMinConfirmations: state.settings.mayOverrideMinConfirmations,
     provider: state.provider,
     chainId,
   };

packages/airnode-node/src/evm/handlers/initialize-provider.ts

@@ -53,6 +53,7 @@ async function fetchTransactionCounts(currentState: ProviderState<EVMProviderSta
     masterHDNode: currentState.masterHDNode,
     provider: currentState.provider,
     minConfirmations: currentState.settings.minConfirmations,
+    mayOverrideMinConfirmations: currentState.settings.mayOverrideMinConfirmations,
   };
   // This should not throw
   const [logs, res] = await transactionCounts.fetchBySponsor(sponsors, fetchOptions);

packages/airnode-node/src/evm/requests/event-logs.test.ts

@@ -54,6 +54,7 @@ describe('EVM event logs - fetch', () => {
       blockHistoryLimit: 300,
       currentBlock: 10716084,
       minConfirmations: 1,
+      mayOverrideMinConfirmations: false,
       provider: new ethers.providers.JsonRpcProvider(),
       chainId: '31137',
     };
@@ -109,6 +110,7 @@ describe('EVM event logs - fetch', () => {
       blockHistoryLimit: 30,
       currentBlock: 10716084,
       minConfirmations: 0,
+      mayOverrideMinConfirmations: false,
       provider: new ethers.providers.JsonRpcProvider(),
       chainId: '31137',
     };
@@ -139,6 +141,7 @@ describe('EVM event logs - fetch', () => {
       blockHistoryLimit: 300,
       currentBlock: 10716084,
       minConfirmations: 0,
+      mayOverrideMinConfirmations: false,
       provider: new ethers.providers.JsonRpcProvider(),
       chainId: '31137',
     };
@@ -154,6 +157,7 @@ describe('EVM event logs - fetch', () => {
       blockHistoryLimit: 99999999,
       currentBlock: 10716084,
       minConfirmations: 10,
+      mayOverrideMinConfirmations: false,
       provider: new ethers.providers.JsonRpcProvider(),
       chainId: '31137',
     };
@@ -177,6 +181,7 @@ describe('EVM event logs - fetch', () => {
       blockHistoryLimit: 30,
       currentBlock: 10716084,
       minConfirmations: 99999999,
+      mayOverrideMinConfirmations: false,
       provider: new ethers.providers.JsonRpcProvider(),
       chainId: '31137',
     };

packages/airnode-node/src/evm/requests/event-logs.ts

@@ -16,6 +16,7 @@ export interface FetchOptions {
   readonly blockHistoryLimit: number;
   readonly currentBlock: number;
   readonly minConfirmations: number;
+  readonly mayOverrideMinConfirmations: boolean;
   readonly provider: ethers.providers.JsonRpcProvider;
   readonly chainId: string;
 }
@@ -37,7 +38,11 @@ export async function fetch(options: FetchOptions): Promise<EVMEventLog[]> {
   // Protect against a potential negative fromBlock value
   const fromBlock = Math.max(0, options.currentBlock - options.blockHistoryLimit);
   // toBlock should always be >= fromBlock
-  const toBlock = Math.max(fromBlock, options.currentBlock - options.minConfirmations);
+  const toBlock = Math.max(
+    fromBlock,
+    // Fetch up to currentBlock to handle possibility of _minConfirmations parameter in request
+    options.currentBlock - (options.mayOverrideMinConfirmations ? 0 : options.minConfirmations)
+  );
 
   const filter: ethers.providers.Filter = {
     fromBlock,

packages/airnode-node/src/evm/transaction-counts.test.ts

@@ -25,6 +25,7 @@ describe('fetchBySponsor', () => {
       masterHDNode: wallet.getMasterHDNode(config),
       provider: new ethers.providers.JsonRpcProvider(),
       minConfirmations: 0,
+      mayOverrideMinConfirmations: false,
     };
     const addresses = ['0x69e2B095fbAc6C3f9E528Ef21882b86BF1595181', '0x69e2B095fbAc6C3f9E528Ef21882b86BF1595181'];
     const [logs, res] = await transactions.fetchBySponsor(addresses, options);
@@ -41,6 +42,7 @@ describe('fetchBySponsor', () => {
       masterHDNode: wallet.getMasterHDNode(config),
       provider: new ethers.providers.JsonRpcProvider(),
       minConfirmations: 2,
+      mayOverrideMinConfirmations: false,
     };
     const addresses = ['0x69e2B095fbAc6C3f9E528Ef21882b86BF1595181', '0x69e2B095fbAc6C3f9E528Ef21882b86BF1595181'];
     const [logs, res] = await transactions.fetchBySponsor(addresses, options);
@@ -53,6 +55,24 @@ describe('fetchBySponsor', () => {
     );
   });
 
+  it('fetches up to current block when minConfirmations may be overridden', async () => {
+    getTransactionCountMock.mockResolvedValueOnce(5);
+    const currentBlock = 10716084;
+    const options = {
+      currentBlock: currentBlock,
+      masterHDNode: wallet.getMasterHDNode(config),
+      provider: new ethers.providers.JsonRpcProvider(),
+      minConfirmations: 2,
+      mayOverrideMinConfirmations: true,
+    };
+    const addresses = ['0x69e2B095fbAc6C3f9E528Ef21882b86BF1595181', '0x69e2B095fbAc6C3f9E528Ef21882b86BF1595181'];
+    const [logs, res] = await transactions.fetchBySponsor(addresses, options);
+    expect(logs).toEqual([]);
+    expect(res).toEqual({ '0x69e2B095fbAc6C3f9E528Ef21882b86BF1595181': 5 });
+    expect(getTransactionCountMock).toHaveBeenCalledTimes(1);
+    expect(getTransactionCountMock).toHaveBeenCalledWith('0xdBFe14C250643DEFE92C9AbC52103bf4978C7113', currentBlock);
+  });
+
   it('returns transaction counts for multiple wallets', async () => {
     getTransactionCountMock.mockResolvedValueOnce(45);
     getTransactionCountMock.mockResolvedValueOnce(123);
@@ -61,6 +81,7 @@ describe('fetchBySponsor', () => {
       masterHDNode: wallet.getMasterHDNode(config),
       provider: new ethers.providers.JsonRpcProvider(),
       minConfirmations: 0,
+      mayOverrideMinConfirmations: false,
     };
     const addresses = ['0x69e2B095fbAc6C3f9E528Ef21882b86BF1595181', '0x99bd3a5A045066F1CEf37A0A952DFa87Af9D898E'];
     const [logs, res] = await transactions.fetchBySponsor(addresses, options);
@@ -84,6 +105,7 @@ describe('fetchBySponsor', () => {
       masterHDNode: wallet.getMasterHDNode(config),
       provider: new ethers.providers.JsonRpcProvider(),
       minConfirmations: 0,
+      mayOverrideMinConfirmations: false,
     };
     const addresses = ['0x69e2B095fbAc6C3f9E528Ef21882b86BF1595181', '0x69e2B095fbAc6C3f9E528Ef21882b86BF1595181'];
     const [logs, res] = await transactions.fetchBySponsor(addresses, options);
@@ -104,6 +126,7 @@ describe('fetchBySponsor', () => {
       masterHDNode: wallet.getMasterHDNode(config),
       provider: new ethers.providers.JsonRpcProvider(),
       minConfirmations: 0,
+      mayOverrideMinConfirmations: false,
     };
     const addresses = ['0x69e2B095fbAc6C3f9E528Ef21882b86BF1595181', '0x69e2B095fbAc6C3f9E528Ef21882b86BF1595181'];
     const [logs, res] = await transactions.fetchBySponsor(addresses, options);

packages/airnode-node/src/evm/transaction-counts.ts

@@ -16,6 +16,7 @@ interface FetchOptions {
   readonly masterHDNode: ethers.utils.HDNode;
   readonly provider: ethers.providers.JsonRpcProvider;
   readonly minConfirmations: number;
+  readonly mayOverrideMinConfirmations: boolean;
 }
 
 async function getWalletTransactionCount(
@@ -24,7 +25,11 @@ async function getWalletTransactionCount(
 ): Promise<LogsData<TransactionCountBySponsorAddress | null>> {
   const address = wallet.deriveSponsorWallet(options.masterHDNode, sponsorAddress).address;
   const operation = () =>
-    options.provider.getTransactionCount(address, options.currentBlock - options.minConfirmations);
+    options.provider.getTransactionCount(
+      address,
+      // Fetch up to currentBlock to handle possibility of _minConfirmations parameter in request
+      options.currentBlock - (options.mayOverrideMinConfirmations ? 0 : options.minConfirmations)
+    );
   const goCount = await go(operation, { retries: 1, attemptTimeoutMs: BLOCKCHAIN_CALL_ATTEMPT_TIMEOUT });
   if (!goCount.success) {
     const log = logger.pend('ERROR', `Unable to fetch transaction count for wallet:${address}`, goCount.error);

packages/airnode-node/src/handlers/start-coordinator.test.ts

@@ -27,9 +27,11 @@ import { ethers } from 'ethers';
 import * as adapter from '@api3/airnode-adapter';
 import * as validator from '@api3/airnode-validator';
 import { randomHexString, caching } from '@api3/airnode-utilities';
-import { startCoordinator } from './start-coordinator';
+import { getMinConfirmationsReservedParameter, startCoordinator } from './start-coordinator';
 import * as fixtures from '../../test/fixtures';
 import * as calls from '../coordinator/calls';
+import { buildAggregatedRegularApiCall, buildConfig } from '../../test/fixtures';
+import { BLOCK_COUNT_HISTORY_LIMIT } from '../constants';
 
 describe('startCoordinator', () => {
   jest.setTimeout(30_000);
@@ -215,3 +217,30 @@ describe('startCoordinator', () => {
     expect(contract.fulfill).not.toHaveBeenCalled();
   });
 });
+
+describe('getMinConfirmationsReservedParameter', () => {
+  const config = buildConfig();
+
+  it('returns _minConfirmations reserved parameter as a number', () => {
+    const aggregatedRegularApiCall = buildAggregatedRegularApiCall();
+    aggregatedRegularApiCall.parameters = { ...aggregatedRegularApiCall.parameters, _minConfirmations: '0' };
+    expect(getMinConfirmationsReservedParameter(aggregatedRegularApiCall, config)).toEqual(0);
+  });
+
+  it('returns undefined for a missing or invalid _minConfirmations reserved parameter', () => {
+    const aggregatedRegularApiCall = buildAggregatedRegularApiCall();
+    expect(getMinConfirmationsReservedParameter(aggregatedRegularApiCall, config)).toBe(undefined);
+
+    aggregatedRegularApiCall.parameters = { ...aggregatedRegularApiCall.parameters, _minConfirmations: '-1' };
+    expect(getMinConfirmationsReservedParameter(aggregatedRegularApiCall, config)).toBe(undefined);
+
+    aggregatedRegularApiCall.parameters = { ...aggregatedRegularApiCall.parameters, _minConfirmations: '34.2' };
+    expect(getMinConfirmationsReservedParameter(aggregatedRegularApiCall, config)).toBe(undefined);
+
+    aggregatedRegularApiCall.parameters = {
+      ...aggregatedRegularApiCall.parameters,
+      _minConfirmations: (BLOCK_COUNT_HISTORY_LIMIT + 1).toString(),
+    };
+    expect(getMinConfirmationsReservedParameter(aggregatedRegularApiCall, config)).toBe(undefined);
+  });
+});