Merge pull request #73 from oasysgames/fix/add_runtime_statis_log

add logs to user can view total time processing
oasysgames · Dec 19, 2023 · 5775341 · 5775341
2 parents 2205ef5 + ba164b9
commit 5775341
Show file tree

Hide file tree

Showing 4 changed files with 119 additions and 111 deletions.
diff --git a/src/execute/executeStakerReward.ts b/src/execute/executeStakerReward.ts
@@ -1,10 +1,10 @@
 import moment = require('moment-timezone');
 import { GoogleSpreadsheet } from 'google-spreadsheet';
 import {
+  exportCsv,
   getAdditionalDataForStakerReward,
   getEpoches,
   getOasPricesForEpoch,
-  handleExport,
 } from '../module/RewardStakes';
 import {
   DataExport,
@@ -14,12 +14,9 @@ import {
   Verse,
   stakerRewardArgs,
 } from '../types';
-import {
-  generateNumberArray,
-  isValidAddresses,
-  sortByTimeStamp,
-} from '../utils';
+import { generateNumberArray, isValidAddresses } from '../utils';
 import { convertAddressesToArray } from '../utils/convert';
+import { getTotalSecondProcess } from '../utils/date';
 import {
   DEFAULT_LIST_PRICE,
   HEADER_FOR_STAKING_REWARD,
@@ -28,6 +25,8 @@ import {
 import { Subgraph } from '../utils/subgraph';
 
 export const main = async (argv: stakerRewardArgs) => {
+  const startTimeProcess = Date.now();
+
   // validate address
   const addresses = convertAddressesToArray(argv.staker_addresses);
   if (!isValidAddresses(addresses)) {
@@ -54,23 +53,10 @@ export const main = async (argv: stakerRewardArgs) => {
     argv,
   );
   // data to export
-  let dataExport: DataExport[] = await getDataExport(
-    prepareData,
-    subgraph,
-    argv,
-  );
+  await handleExport(prepareData, subgraph, argv, header);
 
-  //sort by timestamp
-  dataExport = sortByTimeStamp(dataExport, 'asc');
-
-  // process export
-  await handleExport(
-    dataExport,
-    Boolean(argv.export_csv_online),
-    argv.output,
-    `staker-reward`,
-    header,
-  );
+  const totalSecondsProcess = getTotalSecondProcess(startTimeProcess);
+  console.log(`==> Total: ${totalSecondsProcess} seconds`);
 };
 
 const getHeader = (argv: stakerRewardArgs): string[] => {
@@ -120,44 +106,58 @@ const getPrepareData = async (
     }),
   );
 };
-const getDataExport = async (
+const handleExport = async (
   prepareData: PrepareData[],
   subgraph: Subgraph,
   argv: stakerRewardArgs,
+  header: string[],
 ): Promise<DataExport[]> => {
   // set the address to lowercase
   const addresses = convertAddressesToArray(argv.staker_addresses);
-  const resultsPromise = prepareData.map(async (item: PrepareData) => {
+
+  const results: DataExport[] = [];
+
+  for (const item of prepareData) {
     const { oasPrices, timeData } = item;
     const { block, epoch, timestamp } = timeData;
-    console.log('RUNNING EPOCH ', epoch);
-
-    const validatorResults = await Promise.all(
-      addresses?.map(async (address: string) => {
-        const listStakerStake = await subgraph.getListStakerStake(
-          block,
-          address,
-          epoch,
-        );
-
-        // format data
-        const { rowData } = getAdditionalDataForStakerReward(
-          oasPrices,
-          listStakerStake,
-          timeData,
-          argv.price,
-          address,
-        );
-        return {
-          rowData,
-          timestamp,
-        };
-      }),
-    );
+    const startTimeProcess = Date.now();
+    console.log('PROCESSING WITH EPOCH', epoch);
 
-    return validatorResults;
-  });
+    const promises = addresses?.map(async (address: string) => {
+      const listStakerStake = await subgraph.getListStakerStake(
+        block,
+        address,
+        epoch,
+      );
 
-  const results = await Promise.all(resultsPromise);
-  return results.flat();
+      // format data
+      const { rowData } = getAdditionalDataForStakerReward(
+        oasPrices,
+        listStakerStake,
+        timeData,
+        argv.price,
+        address,
+      );
+      return {
+        rowData,
+        timestamp,
+      };
+    });
+
+    const dataExport = await Promise.all(promises);
+    // process export
+    await exportCsv(
+      dataExport,
+      Boolean(argv.export_csv_online),
+      argv.output,
+      `staker-reward`,
+      header,
+    );
+    results.push(...dataExport);
+    const totalSecondsEpoch = getTotalSecondProcess(startTimeProcess);
+    console.info(
+      `-->Export at Epoch ${epoch} took ${totalSecondsEpoch} seconds`,
+    );
+  }
+  return results;
 };
diff --git a/src/execute/executeValidatorReward.ts b/src/execute/executeValidatorReward.ts
@@ -1,9 +1,9 @@
 import moment = require('moment-timezone');
 import {
+  exportCsv,
   getAdditionalDataForCommissionReward,
   getEpoches,
   getOasPricesForEpoch,
-  handleExport,
 } from '../module/RewardStakes';
 import {
   DataExport,
@@ -13,19 +13,18 @@ import {
   Verse,
   validatorRewardArgs,
 } from '../types';
-import {
-  generateNumberArray,
-  isValidAddresses,
-  sortByTimeStamp,
-} from '../utils';
+import { generateNumberArray, isValidAddresses } from '../utils';
+import { convertAddressesToArray } from '../utils/convert';
+import { getTotalSecondProcess } from '../utils/date';
 import {
   DEFAULT_LIST_PRICE,
   HEADER_FOR_VALIDATOR_REWARD,
 } from '../utils/google';
 import { Subgraph } from '../utils/subgraph';
-import { convertAddressesToArray } from '../utils/convert';
 // main process
 export const main = async (argv: validatorRewardArgs) => {
+  const startTimeProcess = Date.now();
+
   // validate address
   const addresses = convertAddressesToArray(argv.validator_addresses);
   if (!isValidAddresses(addresses)) {
@@ -49,23 +48,10 @@ export const main = async (argv: validatorRewardArgs) => {
   );
 
   // data to export
-  let dataExport: DataExport[] = await getDataExport(
-    prepareData,
-    subgraph,
-    argv,
-  );
-
-  //sort by timestamp
-  dataExport = sortByTimeStamp(dataExport, 'asc');
+  await handleExport(prepareData, subgraph, argv, header);
 
-  // process export
-  await handleExport(
-    dataExport,
-    Boolean(argv.export_csv_online),
-    argv.output,
-    'commission-reward',
-    header,
-  );
+  const totalSecondsProcess = getTotalSecondProcess(startTimeProcess);
+  console.log(`==> Total: ${totalSecondsProcess} seconds`);
 };
 
 const getHeader = (argv: validatorRewardArgs): string[] => {
@@ -117,47 +103,61 @@ const getPrepareData = async (
   );
 };
 
-const getDataExport = async (
+const handleExport = async (
   prepareData: PrepareData[],
   subgraph: Subgraph,
   argv: validatorRewardArgs,
+  header: string[],
 ): Promise<DataExport[]> => {
-  // Set the address to lowercase
   const validator_addresses = convertAddressesToArray(argv.validator_addresses);
 
-  const resultsPromise = prepareData.map(async (item: PrepareData) => {
+  const results: DataExport[] = [];
+
+  for (const item of prepareData) {
     const { oasPrices, timeData } = item;
     const { block, epoch, timestamp } = timeData;
-    console.log('RUNNING EPOCH ', epoch);
-
-    const validatorResults = await Promise.all(
-      validator_addresses?.map(async (address: string) => {
-        const validatorAddress = address;
-        // Get totalStake of validator
-        const validatorStake = await subgraph.getValidatorTotalStake(
-          epoch,
-          block,
-          validatorAddress,
-        );
-
-        // Format data
-        const { rowData } = getAdditionalDataForCommissionReward(
-          oasPrices,
-          validatorStake,
-          timeData,
-          argv.price,
-          validatorAddress,
-        );
-        return {
-          rowData,
-          timestamp,
-        };
-      }),
-    );
+    const startTimeProcess = Date.now();
+    console.log('PROCESSING WITH EPOCH', epoch);
 
-    return validatorResults;
-  });
+    const promises = validator_addresses.map(async (address: string) => {
+      const validatorAddress = address;
+      const validatorStake = await subgraph.getValidatorTotalStake(
+        epoch,
+        block,
+        validatorAddress,
+      );
+
+      const { rowData } = getAdditionalDataForCommissionReward(
+        oasPrices,
+        validatorStake,
+        timeData,
+        argv.price,
+        validatorAddress,
+      );
+
+      return {
+        rowData,
+        timestamp,
+      };
+    });
+
+    const dataExport = await Promise.all(promises);
+    //sort by timestamp
+
+    // process export
+    await exportCsv(
+      dataExport,
+      Boolean(argv.export_csv_online),
+      argv.output,
+      'commission-reward',
+      header,
+    );
+    results.push(...dataExport);
+    const totalSecondsEpoch = getTotalSecondProcess(startTimeProcess);
+    console.info(
+      `-->Export at Epoch ${epoch} took ${totalSecondsEpoch} seconds`,
+    );
+  }
 
-  const results = await Promise.all(resultsPromise);
-  return results.flat();
+  return results;
 };
diff --git a/src/module/RewardStakes.ts b/src/module/RewardStakes.ts
@@ -149,13 +149,15 @@ export const exportCsvLocal = async (
   );
 };
 
-export const handleExport = async (
+export const exportCsv = async (
   array: any,
   isOnline: boolean,
   output: string,
   fileName: string,
   header: string[],
-) => {
+): Promise<boolean> => {
+  // console.info(`Start handle export`);
+
   let doc: GoogleSpreadsheet;
   if (isOnline) {
     doc = await getSpreadSheet();
@@ -167,8 +169,10 @@ export const handleExport = async (
     isOnline
       ? await exportCsvOnline(doc, rowData, timestamp, header)
       : await exportCsvLocal(rowData, header, fileName, output);
-    await sleep(1500);
+    // await sleep(1500);
   }
+  // console.log('Export process complete!');
+  return true;
 };
 
 export const getAdditionalDataForCommissionReward = (

diff --git a/src/utils/date.ts b/src/utils/date.ts
@@ -19,3 +19,7 @@ export const getTime = (timestamp: Date) => {
 export const getMonthDate = (timestamp: Date) => {
   return dateParts(timestamp).slice(0, 2).join('');
 };
+
+export const getTotalSecondProcess = (startTimeProcess: number) => {
+  return ((Date.now() - startTimeProcess) / 1000).toFixed(2);
+};