From d970989d0e72f448066ce2fae90763c84ad11324 Mon Sep 17 00:00:00 2001 From: longpv Date: Mon, 18 Dec 2023 12:52:25 +0700 Subject: [PATCH 1/2] add logs to user can view total time processing --- src/execute/executeStakerReward.ts | 71 +++++++++++++----------- src/execute/executeValidatorReward.ts | 77 +++++++++++++++------------ src/module/RewardStakes.ts | 4 ++ src/utils/date.ts | 4 ++ 4 files changed, 90 insertions(+), 66 deletions(-) diff --git a/src/execute/executeStakerReward.ts b/src/execute/executeStakerReward.ts index 68ed479..86d78b7 100755 --- a/src/execute/executeStakerReward.ts +++ b/src/execute/executeStakerReward.ts @@ -20,6 +20,7 @@ import { sortByTimeStamp, } from '../utils'; import { convertAddressesToArray } from '../utils/convert'; +import { getTotalSecondProcess } from '../utils/date'; import { DEFAULT_LIST_PRICE, HEADER_FOR_STAKING_REWARD, @@ -28,6 +29,8 @@ import { import { Subgraph } from '../utils/subgraph'; export const main = async (argv: stakerRewardArgs) => { + const startTimeProcess = Date.now(); + // validate address const addresses = convertAddressesToArray(argv.staker_addresses); if (!isValidAddresses(addresses)) { @@ -71,6 +74,8 @@ export const main = async (argv: stakerRewardArgs) => { `staker-reward`, header, ); + const totalSecondsProcess = getTotalSecondProcess(startTimeProcess); + console.log(`==> ${totalSecondsProcess} seconds`); }; const getHeader = (argv: stakerRewardArgs): string[] => { @@ -127,37 +132,41 @@ const getDataExport = async ( ): Promise => { // set the address to lowercase const addresses = convertAddressesToArray(argv.staker_addresses); - const resultsPromise = prepareData.map(async (item: PrepareData) => { + + const results: DataExport[] = []; + + for (const item of prepareData) { const { oasPrices, timeData } = item; const { block, epoch, timestamp } = timeData; - console.log('RUNNING EPOCH ', epoch); - - const validatorResults = await Promise.all( - addresses?.map(async (address: string) => { - const listStakerStake = await subgraph.getListStakerStake( - block, - address, - epoch, - ); - - // format data - const { rowData } = getAdditionalDataForStakerReward( - oasPrices, - listStakerStake, - timeData, - argv.price, - address, - ); - return { - rowData, - timestamp, - }; - }), - ); - - return validatorResults; - }); - - const results = await Promise.all(resultsPromise); - return results.flat(); + const startTimeProcess = Date.now(); + console.log('PROCESSING WITH EPOCH', epoch); + + const promises = addresses?.map(async (address: string) => { + const listStakerStake = await subgraph.getListStakerStake( + block, + address, + epoch, + ); + + // format data + const { rowData } = getAdditionalDataForStakerReward( + oasPrices, + listStakerStake, + timeData, + argv.price, + address, + ); + return { + rowData, + timestamp, + }; + }); + + const stakerResults = await Promise.all(promises); + + results.push(...stakerResults); + const totalSecondsEpoch = getTotalSecondProcess(startTimeProcess); + console.info(`--> Epoch ${epoch} took ${totalSecondsEpoch} seconds`); + } + return results; }; diff --git a/src/execute/executeValidatorReward.ts b/src/execute/executeValidatorReward.ts index 596c5bb..c4906c0 100755 --- a/src/execute/executeValidatorReward.ts +++ b/src/execute/executeValidatorReward.ts @@ -18,14 +18,17 @@ import { isValidAddresses, sortByTimeStamp, } from '../utils'; +import { convertAddressesToArray } from '../utils/convert'; +import { getTotalSecondProcess } from '../utils/date'; import { DEFAULT_LIST_PRICE, HEADER_FOR_VALIDATOR_REWARD, } from '../utils/google'; import { Subgraph } from '../utils/subgraph'; -import { convertAddressesToArray } from '../utils/convert'; // main process export const main = async (argv: validatorRewardArgs) => { + const startTimeProcess = Date.now(); + // validate address const addresses = convertAddressesToArray(argv.validator_addresses); if (!isValidAddresses(addresses)) { @@ -66,6 +69,8 @@ export const main = async (argv: validatorRewardArgs) => { 'commission-reward', header, ); + const totalSecondsProcess = getTotalSecondProcess(startTimeProcess); + console.log(`==> ${totalSecondsProcess} seconds`); }; const getHeader = (argv: validatorRewardArgs): string[] => { @@ -122,42 +127,44 @@ const getDataExport = async ( subgraph: Subgraph, argv: validatorRewardArgs, ): Promise => { - // Set the address to lowercase const validator_addresses = convertAddressesToArray(argv.validator_addresses); - const resultsPromise = prepareData.map(async (item: PrepareData) => { + const results: DataExport[] = []; + + for (const item of prepareData) { const { oasPrices, timeData } = item; const { block, epoch, timestamp } = timeData; - console.log('RUNNING EPOCH ', epoch); - - const validatorResults = await Promise.all( - validator_addresses?.map(async (address: string) => { - const validatorAddress = address; - // Get totalStake of validator - const validatorStake = await subgraph.getValidatorTotalStake( - epoch, - block, - validatorAddress, - ); - - // Format data - const { rowData } = getAdditionalDataForCommissionReward( - oasPrices, - validatorStake, - timeData, - argv.price, - validatorAddress, - ); - return { - rowData, - timestamp, - }; - }), - ); - - return validatorResults; - }); - - const results = await Promise.all(resultsPromise); - return results.flat(); + const startTimeProcess = Date.now(); + console.log('PROCESSING WITH EPOCH', epoch); + + const promises = validator_addresses.map(async (address: string) => { + const validatorAddress = address; + const validatorStake = await subgraph.getValidatorTotalStake( + epoch, + block, + validatorAddress, + ); + + const { rowData } = getAdditionalDataForCommissionReward( + oasPrices, + validatorStake, + timeData, + argv.price, + validatorAddress, + ); + + return { + rowData, + timestamp, + }; + }); + + const validatorResults = await Promise.all(promises); + + results.push(...validatorResults); + const totalSecondsEpoch = getTotalSecondProcess(startTimeProcess); + console.info(`--> Epoch ${epoch} took ${totalSecondsEpoch} seconds`); + } + + return results; }; diff --git a/src/module/RewardStakes.ts b/src/module/RewardStakes.ts index 99aab87..e21ba77 100755 --- a/src/module/RewardStakes.ts +++ b/src/module/RewardStakes.ts @@ -156,6 +156,8 @@ export const handleExport = async ( fileName: string, header: string[], ) => { + console.info(`Start handle export`); + let doc: GoogleSpreadsheet; if (isOnline) { doc = await getSpreadSheet(); @@ -168,7 +170,9 @@ export const handleExport = async ( ? await exportCsvOnline(doc, rowData, timestamp, header) : await exportCsvLocal(rowData, header, fileName, output); await sleep(1500); + console.log(`Exported: ${i + 1}/${array.length}`); } + console.log('Export process complete!'); }; export const getAdditionalDataForCommissionReward = ( diff --git a/src/utils/date.ts b/src/utils/date.ts index c5ac784..e21a8a0 100755 --- a/src/utils/date.ts +++ b/src/utils/date.ts @@ -19,3 +19,7 @@ export const getTime = (timestamp: Date) => { export const getMonthDate = (timestamp: Date) => { return dateParts(timestamp).slice(0, 2).join(''); }; + +export const getTotalSecondProcess = (startTimeProcess: number) => { + return ((Date.now() - startTimeProcess) / 1000).toFixed(2); +}; From ba164b9bc708961fd1151ab566f74a7cf7b85257 Mon Sep 17 00:00:00 2001 From: longpv Date: Tue, 19 Dec 2023 09:06:07 +0700 Subject: [PATCH 2/2] optimize speed export --- src/execute/executeStakerReward.ts | 47 +++++++++++-------------- src/execute/executeValidatorReward.ts | 49 ++++++++++++--------------- src/module/RewardStakes.ts | 12 +++---- 3 files changed, 46 insertions(+), 62 deletions(-) diff --git a/src/execute/executeStakerReward.ts b/src/execute/executeStakerReward.ts index 86d78b7..377d72a 100755 --- a/src/execute/executeStakerReward.ts +++ b/src/execute/executeStakerReward.ts @@ -1,10 +1,10 @@ import moment = require('moment-timezone'); import { GoogleSpreadsheet } from 'google-spreadsheet'; import { + exportCsv, getAdditionalDataForStakerReward, getEpoches, getOasPricesForEpoch, - handleExport, } from '../module/RewardStakes'; import { DataExport, @@ -14,11 +14,7 @@ import { Verse, stakerRewardArgs, } from '../types'; -import { - generateNumberArray, - isValidAddresses, - sortByTimeStamp, -} from '../utils'; +import { generateNumberArray, isValidAddresses } from '../utils'; import { convertAddressesToArray } from '../utils/convert'; import { getTotalSecondProcess } from '../utils/date'; import { @@ -57,25 +53,10 @@ export const main = async (argv: stakerRewardArgs) => { argv, ); // data to export - let dataExport: DataExport[] = await getDataExport( - prepareData, - subgraph, - argv, - ); + await handleExport(prepareData, subgraph, argv, header); - //sort by timestamp - dataExport = sortByTimeStamp(dataExport, 'asc'); - - // process export - await handleExport( - dataExport, - Boolean(argv.export_csv_online), - argv.output, - `staker-reward`, - header, - ); const totalSecondsProcess = getTotalSecondProcess(startTimeProcess); - console.log(`==> ${totalSecondsProcess} seconds`); + console.log(`==> Total: ${totalSecondsProcess} seconds`); }; const getHeader = (argv: stakerRewardArgs): string[] => { @@ -125,10 +106,11 @@ const getPrepareData = async ( }), ); }; -const getDataExport = async ( +const handleExport = async ( prepareData: PrepareData[], subgraph: Subgraph, argv: stakerRewardArgs, + header: string[], ): Promise => { // set the address to lowercase const addresses = convertAddressesToArray(argv.staker_addresses); @@ -162,11 +144,20 @@ const getDataExport = async ( }; }); - const stakerResults = await Promise.all(promises); - - results.push(...stakerResults); + const dataExport = await Promise.all(promises); + // process export + await exportCsv( + dataExport, + Boolean(argv.export_csv_online), + argv.output, + `staker-reward`, + header, + ); + results.push(...dataExport); const totalSecondsEpoch = getTotalSecondProcess(startTimeProcess); - console.info(`--> Epoch ${epoch} took ${totalSecondsEpoch} seconds`); + console.info( + `-->Export at Epoch ${epoch} took ${totalSecondsEpoch} seconds`, + ); } return results; }; diff --git a/src/execute/executeValidatorReward.ts b/src/execute/executeValidatorReward.ts index c4906c0..ec64bb5 100755 --- a/src/execute/executeValidatorReward.ts +++ b/src/execute/executeValidatorReward.ts @@ -1,9 +1,9 @@ import moment = require('moment-timezone'); import { + exportCsv, getAdditionalDataForCommissionReward, getEpoches, getOasPricesForEpoch, - handleExport, } from '../module/RewardStakes'; import { DataExport, @@ -13,11 +13,7 @@ import { Verse, validatorRewardArgs, } from '../types'; -import { - generateNumberArray, - isValidAddresses, - sortByTimeStamp, -} from '../utils'; +import { generateNumberArray, isValidAddresses } from '../utils'; import { convertAddressesToArray } from '../utils/convert'; import { getTotalSecondProcess } from '../utils/date'; import { @@ -52,25 +48,10 @@ export const main = async (argv: validatorRewardArgs) => { ); // data to export - let dataExport: DataExport[] = await getDataExport( - prepareData, - subgraph, - argv, - ); + await handleExport(prepareData, subgraph, argv, header); - //sort by timestamp - dataExport = sortByTimeStamp(dataExport, 'asc'); - - // process export - await handleExport( - dataExport, - Boolean(argv.export_csv_online), - argv.output, - 'commission-reward', - header, - ); const totalSecondsProcess = getTotalSecondProcess(startTimeProcess); - console.log(`==> ${totalSecondsProcess} seconds`); + console.log(`==> Total: ${totalSecondsProcess} seconds`); }; const getHeader = (argv: validatorRewardArgs): string[] => { @@ -122,10 +103,11 @@ const getPrepareData = async ( ); }; -const getDataExport = async ( +const handleExport = async ( prepareData: PrepareData[], subgraph: Subgraph, argv: validatorRewardArgs, + header: string[], ): Promise => { const validator_addresses = convertAddressesToArray(argv.validator_addresses); @@ -159,11 +141,22 @@ const getDataExport = async ( }; }); - const validatorResults = await Promise.all(promises); - - results.push(...validatorResults); + const dataExport = await Promise.all(promises); + //sort by timestamp + + // process export + await exportCsv( + dataExport, + Boolean(argv.export_csv_online), + argv.output, + 'commission-reward', + header, + ); + results.push(...dataExport); const totalSecondsEpoch = getTotalSecondProcess(startTimeProcess); - console.info(`--> Epoch ${epoch} took ${totalSecondsEpoch} seconds`); + console.info( + `-->Export at Epoch ${epoch} took ${totalSecondsEpoch} seconds`, + ); } return results; diff --git a/src/module/RewardStakes.ts b/src/module/RewardStakes.ts index e21ba77..d7561dc 100755 --- a/src/module/RewardStakes.ts +++ b/src/module/RewardStakes.ts @@ -149,14 +149,14 @@ export const exportCsvLocal = async ( ); }; -export const handleExport = async ( +export const exportCsv = async ( array: any, isOnline: boolean, output: string, fileName: string, header: string[], -) => { - console.info(`Start handle export`); +): Promise => { + // console.info(`Start handle export`); let doc: GoogleSpreadsheet; if (isOnline) { @@ -169,10 +169,10 @@ export const handleExport = async ( isOnline ? await exportCsvOnline(doc, rowData, timestamp, header) : await exportCsvLocal(rowData, header, fileName, output); - await sleep(1500); - console.log(`Exported: ${i + 1}/${array.length}`); + // await sleep(1500); } - console.log('Export process complete!'); + // console.log('Export process complete!'); + return true; }; export const getAdditionalDataForCommissionReward = (