diff --git a/src/execute/executeStakerReward.ts b/src/execute/executeStakerReward.ts index 68ed479..377d72a 100755 --- a/src/execute/executeStakerReward.ts +++ b/src/execute/executeStakerReward.ts @@ -1,10 +1,10 @@ import moment = require('moment-timezone'); import { GoogleSpreadsheet } from 'google-spreadsheet'; import { + exportCsv, getAdditionalDataForStakerReward, getEpoches, getOasPricesForEpoch, - handleExport, } from '../module/RewardStakes'; import { DataExport, @@ -14,12 +14,9 @@ import { Verse, stakerRewardArgs, } from '../types'; -import { - generateNumberArray, - isValidAddresses, - sortByTimeStamp, -} from '../utils'; +import { generateNumberArray, isValidAddresses } from '../utils'; import { convertAddressesToArray } from '../utils/convert'; +import { getTotalSecondProcess } from '../utils/date'; import { DEFAULT_LIST_PRICE, HEADER_FOR_STAKING_REWARD, @@ -28,6 +25,8 @@ import { import { Subgraph } from '../utils/subgraph'; export const main = async (argv: stakerRewardArgs) => { + const startTimeProcess = Date.now(); + // validate address const addresses = convertAddressesToArray(argv.staker_addresses); if (!isValidAddresses(addresses)) { @@ -54,23 +53,10 @@ export const main = async (argv: stakerRewardArgs) => { argv, ); // data to export - let dataExport: DataExport[] = await getDataExport( - prepareData, - subgraph, - argv, - ); + await handleExport(prepareData, subgraph, argv, header); - //sort by timestamp - dataExport = sortByTimeStamp(dataExport, 'asc'); - - // process export - await handleExport( - dataExport, - Boolean(argv.export_csv_online), - argv.output, - `staker-reward`, - header, - ); + const totalSecondsProcess = getTotalSecondProcess(startTimeProcess); + console.log(`==> Total: ${totalSecondsProcess} seconds`); }; const getHeader = (argv: stakerRewardArgs): string[] => { @@ -120,44 +106,58 @@ const getPrepareData = async ( }), ); }; -const getDataExport = async ( +const handleExport = async ( prepareData: PrepareData[], subgraph: Subgraph, argv: stakerRewardArgs, + header: string[], ): Promise => { // set the address to lowercase const addresses = convertAddressesToArray(argv.staker_addresses); - const resultsPromise = prepareData.map(async (item: PrepareData) => { + + const results: DataExport[] = []; + + for (const item of prepareData) { const { oasPrices, timeData } = item; const { block, epoch, timestamp } = timeData; - console.log('RUNNING EPOCH ', epoch); - - const validatorResults = await Promise.all( - addresses?.map(async (address: string) => { - const listStakerStake = await subgraph.getListStakerStake( - block, - address, - epoch, - ); - - // format data - const { rowData } = getAdditionalDataForStakerReward( - oasPrices, - listStakerStake, - timeData, - argv.price, - address, - ); - return { - rowData, - timestamp, - }; - }), - ); + const startTimeProcess = Date.now(); + console.log('PROCESSING WITH EPOCH', epoch); - return validatorResults; - }); + const promises = addresses?.map(async (address: string) => { + const listStakerStake = await subgraph.getListStakerStake( + block, + address, + epoch, + ); - const results = await Promise.all(resultsPromise); - return results.flat(); + // format data + const { rowData } = getAdditionalDataForStakerReward( + oasPrices, + listStakerStake, + timeData, + argv.price, + address, + ); + return { + rowData, + timestamp, + }; + }); + + const dataExport = await Promise.all(promises); + // process export + await exportCsv( + dataExport, + Boolean(argv.export_csv_online), + argv.output, + `staker-reward`, + header, + ); + results.push(...dataExport); + const totalSecondsEpoch = getTotalSecondProcess(startTimeProcess); + console.info( + `-->Export at Epoch ${epoch} took ${totalSecondsEpoch} seconds`, + ); + } + return results; }; diff --git a/src/execute/executeValidatorReward.ts b/src/execute/executeValidatorReward.ts index 596c5bb..ec64bb5 100755 --- a/src/execute/executeValidatorReward.ts +++ b/src/execute/executeValidatorReward.ts @@ -1,9 +1,9 @@ import moment = require('moment-timezone'); import { + exportCsv, getAdditionalDataForCommissionReward, getEpoches, getOasPricesForEpoch, - handleExport, } from '../module/RewardStakes'; import { DataExport, @@ -13,19 +13,18 @@ import { Verse, validatorRewardArgs, } from '../types'; -import { - generateNumberArray, - isValidAddresses, - sortByTimeStamp, -} from '../utils'; +import { generateNumberArray, isValidAddresses } from '../utils'; +import { convertAddressesToArray } from '../utils/convert'; +import { getTotalSecondProcess } from '../utils/date'; import { DEFAULT_LIST_PRICE, HEADER_FOR_VALIDATOR_REWARD, } from '../utils/google'; import { Subgraph } from '../utils/subgraph'; -import { convertAddressesToArray } from '../utils/convert'; // main process export const main = async (argv: validatorRewardArgs) => { + const startTimeProcess = Date.now(); + // validate address const addresses = convertAddressesToArray(argv.validator_addresses); if (!isValidAddresses(addresses)) { @@ -49,23 +48,10 @@ export const main = async (argv: validatorRewardArgs) => { ); // data to export - let dataExport: DataExport[] = await getDataExport( - prepareData, - subgraph, - argv, - ); - - //sort by timestamp - dataExport = sortByTimeStamp(dataExport, 'asc'); + await handleExport(prepareData, subgraph, argv, header); - // process export - await handleExport( - dataExport, - Boolean(argv.export_csv_online), - argv.output, - 'commission-reward', - header, - ); + const totalSecondsProcess = getTotalSecondProcess(startTimeProcess); + console.log(`==> Total: ${totalSecondsProcess} seconds`); }; const getHeader = (argv: validatorRewardArgs): string[] => { @@ -117,47 +103,61 @@ const getPrepareData = async ( ); }; -const getDataExport = async ( +const handleExport = async ( prepareData: PrepareData[], subgraph: Subgraph, argv: validatorRewardArgs, + header: string[], ): Promise => { - // Set the address to lowercase const validator_addresses = convertAddressesToArray(argv.validator_addresses); - const resultsPromise = prepareData.map(async (item: PrepareData) => { + const results: DataExport[] = []; + + for (const item of prepareData) { const { oasPrices, timeData } = item; const { block, epoch, timestamp } = timeData; - console.log('RUNNING EPOCH ', epoch); - - const validatorResults = await Promise.all( - validator_addresses?.map(async (address: string) => { - const validatorAddress = address; - // Get totalStake of validator - const validatorStake = await subgraph.getValidatorTotalStake( - epoch, - block, - validatorAddress, - ); - - // Format data - const { rowData } = getAdditionalDataForCommissionReward( - oasPrices, - validatorStake, - timeData, - argv.price, - validatorAddress, - ); - return { - rowData, - timestamp, - }; - }), - ); + const startTimeProcess = Date.now(); + console.log('PROCESSING WITH EPOCH', epoch); - return validatorResults; - }); + const promises = validator_addresses.map(async (address: string) => { + const validatorAddress = address; + const validatorStake = await subgraph.getValidatorTotalStake( + epoch, + block, + validatorAddress, + ); + + const { rowData } = getAdditionalDataForCommissionReward( + oasPrices, + validatorStake, + timeData, + argv.price, + validatorAddress, + ); + + return { + rowData, + timestamp, + }; + }); + + const dataExport = await Promise.all(promises); + //sort by timestamp + + // process export + await exportCsv( + dataExport, + Boolean(argv.export_csv_online), + argv.output, + 'commission-reward', + header, + ); + results.push(...dataExport); + const totalSecondsEpoch = getTotalSecondProcess(startTimeProcess); + console.info( + `-->Export at Epoch ${epoch} took ${totalSecondsEpoch} seconds`, + ); + } - const results = await Promise.all(resultsPromise); - return results.flat(); + return results; }; diff --git a/src/module/RewardStakes.ts b/src/module/RewardStakes.ts index 99aab87..d7561dc 100755 --- a/src/module/RewardStakes.ts +++ b/src/module/RewardStakes.ts @@ -149,13 +149,15 @@ export const exportCsvLocal = async ( ); }; -export const handleExport = async ( +export const exportCsv = async ( array: any, isOnline: boolean, output: string, fileName: string, header: string[], -) => { +): Promise => { + // console.info(`Start handle export`); + let doc: GoogleSpreadsheet; if (isOnline) { doc = await getSpreadSheet(); @@ -167,8 +169,10 @@ export const handleExport = async ( isOnline ? await exportCsvOnline(doc, rowData, timestamp, header) : await exportCsvLocal(rowData, header, fileName, output); - await sleep(1500); + // await sleep(1500); } + // console.log('Export process complete!'); + return true; }; export const getAdditionalDataForCommissionReward = ( diff --git a/src/utils/date.ts b/src/utils/date.ts index c5ac784..e21a8a0 100755 --- a/src/utils/date.ts +++ b/src/utils/date.ts @@ -19,3 +19,7 @@ export const getTime = (timestamp: Date) => { export const getMonthDate = (timestamp: Date) => { return dateParts(timestamp).slice(0, 2).join(''); }; + +export const getTotalSecondProcess = (startTimeProcess: number) => { + return ((Date.now() - startTimeProcess) / 1000).toFixed(2); +};