optimize speed export

oasysgames · Dec 19, 2023 · ba164b9 · ba164b9
1 parent d970989
commit ba164b9
Show file tree

Hide file tree

Showing 3 changed files with 46 additions and 62 deletions.
diff --git a/src/execute/executeStakerReward.ts b/src/execute/executeStakerReward.ts
@@ -1,10 +1,10 @@
 import moment = require('moment-timezone');
 import { GoogleSpreadsheet } from 'google-spreadsheet';
 import {
+  exportCsv,
   getAdditionalDataForStakerReward,
   getEpoches,
   getOasPricesForEpoch,
-  handleExport,
 } from '../module/RewardStakes';
 import {
   DataExport,
@@ -14,11 +14,7 @@ import {
   Verse,
   stakerRewardArgs,
 } from '../types';
-import {
-  generateNumberArray,
-  isValidAddresses,
-  sortByTimeStamp,
-} from '../utils';
+import { generateNumberArray, isValidAddresses } from '../utils';
 import { convertAddressesToArray } from '../utils/convert';
 import { getTotalSecondProcess } from '../utils/date';
 import {
@@ -57,25 +53,10 @@ export const main = async (argv: stakerRewardArgs) => {
     argv,
   );
   // data to export
-  let dataExport: DataExport[] = await getDataExport(
-    prepareData,
-    subgraph,
-    argv,
-  );
+  await handleExport(prepareData, subgraph, argv, header);
 
-  //sort by timestamp
-  dataExport = sortByTimeStamp(dataExport, 'asc');
-
-  // process export
-  await handleExport(
-    dataExport,
-    Boolean(argv.export_csv_online),
-    argv.output,
-    `staker-reward`,
-    header,
-  );
   const totalSecondsProcess = getTotalSecondProcess(startTimeProcess);
-  console.log(`==> ${totalSecondsProcess} seconds`);
+  console.log(`==> Total: ${totalSecondsProcess} seconds`);
 };
 
 const getHeader = (argv: stakerRewardArgs): string[] => {
@@ -125,10 +106,11 @@ const getPrepareData = async (
     }),
   );
 };
-const getDataExport = async (
+const handleExport = async (
   prepareData: PrepareData[],
   subgraph: Subgraph,
   argv: stakerRewardArgs,
+  header: string[],
 ): Promise<DataExport[]> => {
   // set the address to lowercase
   const addresses = convertAddressesToArray(argv.staker_addresses);
@@ -162,11 +144,20 @@ const getDataExport = async (
       };
     });
 
-    const stakerResults = await Promise.all(promises);
-
-    results.push(...stakerResults);
+    const dataExport = await Promise.all(promises);
+    // process export
+    await exportCsv(
+      dataExport,
+      Boolean(argv.export_csv_online),
+      argv.output,
+      `staker-reward`,
+      header,
+    );
+    results.push(...dataExport);
     const totalSecondsEpoch = getTotalSecondProcess(startTimeProcess);
-    console.info(`--> Epoch ${epoch} took ${totalSecondsEpoch} seconds`);
+    console.info(
+      `-->Export at Epoch ${epoch} took ${totalSecondsEpoch} seconds`,
+    );
   }
   return results;
 };
diff --git a/src/execute/executeValidatorReward.ts b/src/execute/executeValidatorReward.ts
@@ -1,9 +1,9 @@
 import moment = require('moment-timezone');
 import {
+  exportCsv,
   getAdditionalDataForCommissionReward,
   getEpoches,
   getOasPricesForEpoch,
-  handleExport,
 } from '../module/RewardStakes';
 import {
   DataExport,
@@ -13,11 +13,7 @@ import {
   Verse,
   validatorRewardArgs,
 } from '../types';
-import {
-  generateNumberArray,
-  isValidAddresses,
-  sortByTimeStamp,
-} from '../utils';
+import { generateNumberArray, isValidAddresses } from '../utils';
 import { convertAddressesToArray } from '../utils/convert';
 import { getTotalSecondProcess } from '../utils/date';
 import {
@@ -52,25 +48,10 @@ export const main = async (argv: validatorRewardArgs) => {
   );
 
   // data to export
-  let dataExport: DataExport[] = await getDataExport(
-    prepareData,
-    subgraph,
-    argv,
-  );
+  await handleExport(prepareData, subgraph, argv, header);
 
-  //sort by timestamp
-  dataExport = sortByTimeStamp(dataExport, 'asc');
-
-  // process export
-  await handleExport(
-    dataExport,
-    Boolean(argv.export_csv_online),
-    argv.output,
-    'commission-reward',
-    header,
-  );
   const totalSecondsProcess = getTotalSecondProcess(startTimeProcess);
-  console.log(`==> ${totalSecondsProcess} seconds`);
+  console.log(`==> Total: ${totalSecondsProcess} seconds`);
 };
 
 const getHeader = (argv: validatorRewardArgs): string[] => {
@@ -122,10 +103,11 @@ const getPrepareData = async (
   );
 };
 
-const getDataExport = async (
+const handleExport = async (
   prepareData: PrepareData[],
   subgraph: Subgraph,
   argv: validatorRewardArgs,
+  header: string[],
 ): Promise<DataExport[]> => {
   const validator_addresses = convertAddressesToArray(argv.validator_addresses);
 
@@ -159,11 +141,22 @@ const getDataExport = async (
       };
     });
 
-    const validatorResults = await Promise.all(promises);
-
-    results.push(...validatorResults);
+    const dataExport = await Promise.all(promises);
+    //sort by timestamp
+
+    // process export
+    await exportCsv(
+      dataExport,
+      Boolean(argv.export_csv_online),
+      argv.output,
+      'commission-reward',
+      header,
+    );
+    results.push(...dataExport);
     const totalSecondsEpoch = getTotalSecondProcess(startTimeProcess);
-    console.info(`--> Epoch ${epoch} took ${totalSecondsEpoch} seconds`);
+    console.info(
+      `-->Export at Epoch ${epoch} took ${totalSecondsEpoch} seconds`,
+    );
   }
 
   return results;

diff --git a/src/module/RewardStakes.ts b/src/module/RewardStakes.ts
@@ -149,14 +149,14 @@ export const exportCsvLocal = async (
   );
 };
 
-export const handleExport = async (
+export const exportCsv = async (
   array: any,
   isOnline: boolean,
   output: string,
   fileName: string,
   header: string[],
-) => {
-  console.info(`Start handle export`);
+): Promise<boolean> => {
+  // console.info(`Start handle export`);
 
   let doc: GoogleSpreadsheet;
   if (isOnline) {
@@ -169,10 +169,10 @@ export const handleExport = async (
     isOnline
       ? await exportCsvOnline(doc, rowData, timestamp, header)
       : await exportCsvLocal(rowData, header, fileName, output);
-    await sleep(1500);
-    console.log(`Exported: ${i + 1}/${array.length}`);
+    // await sleep(1500);
   }
-  console.log('Export process complete!');
+  // console.log('Export process complete!');
+  return true;
 };
 
 export const getAdditionalDataForCommissionReward = (