Merge pull request #63 from gentlementlegen/feat/exponential-backoff

feat: exponential backoff on retrieval failure

README.md

@@ -55,6 +55,9 @@ with:
     evmNetworkId: 100
     evmPrivateEncrypted: "encrypted-key"
     erc20RewardToken: "0xe91D153E0b41518A2Ce8Dd3D7944Fa863463a97d"
+    dataCollection:
+      maxAttempts: 10
+      delayMs: 10000
     incentives:
       enabled: true
       requirePriceLabel: true

package.json

@@ -41,6 +41,7 @@
     "lodash": "4.17.21",
     "markdown-it": "14.1.0",
     "openai": "4.29.1",
+    "ts-retry": "4.2.5",
     "tsx": "4.7.1",
     "typebox-validators": "0.3.5",
     "yaml": "2.4.1"

src/configuration/data-collection-config.ts

@@ -0,0 +1,14 @@
+import { Static, Type } from "@sinclair/typebox";
+
+export const dataCollectionConfigurationType = Type.Object({
+  /**
+   * The maximum amount of retries on failure.
+   */
+  maxAttempts: Type.Number({ default: 10, minimum: 1 }),
+  /**
+   * The delay between each retry, in milliseconds.
+   */
+  delayMs: Type.Number({ default: 10000, minimum: 100 }),
+});
+
+export type DataCollectionConfiguration = Static<typeof dataCollectionConfigurationType>;

src/configuration/incentives.ts

@@ -1,6 +1,7 @@
 import { StaticDecode, Type as T } from "@sinclair/typebox";
 import { StandardValidator } from "typebox-validators";
 import { contentEvaluatorConfigurationType } from "./content-evaluator-config";
+import { dataCollectionConfigurationType } from "./data-collection-config";
 import { dataPurgeConfigurationType } from "./data-purge-config";
 import { formattingEvaluatorConfigurationType } from "./formatting-evaluator-config";
 import { githubCommentConfigurationType } from "./github-comment-config";
@@ -40,7 +41,9 @@ export const incentivesConfigurationSchema = T.Object({
     permitGeneration: permitGenerationConfigurationType,
     githubComment: githubCommentConfigurationType,
   }),
+  dataCollection: dataCollectionConfigurationType,
 });
+
 export const validateIncentivesConfiguration = new StandardValidator(incentivesConfigurationSchema);
 
 export type IncentivesConfiguration = StaticDecode<typeof incentivesConfigurationSchema>;

src/issue-activity.ts

@@ -1,4 +1,7 @@
+import { retryAsyncUntilDefinedDecorator } from "ts-retry";
 import { CommentType } from "./configuration/comment-types";
+import configuration from "./configuration/config-reader";
+import { DataCollectionConfiguration } from "./configuration/data-collection-config";
 import { collectLinkedMergedPulls } from "./data-collection/collect-linked-pulls";
 import {
   GitHubIssue,
@@ -8,18 +11,22 @@ import {
   GitHubPullRequestReviewComment,
   GitHubPullRequestReviewState,
 } from "./github-types";
+import githubCommentModuleInstance from "./helpers/github-comment-module-instance";
+import logger from "./helpers/logger";
 import {
-  IssueParams,
-  PullParams,
   getIssue,
   getIssueComments,
   getIssueEvents,
   getPullRequest,
   getPullRequestReviewComments,
   getPullRequestReviews,
+  IssueParams,
+  PullParams,
 } from "./start";
 
 export class IssueActivity {
+  readonly _configuration: DataCollectionConfiguration = configuration.dataCollection;
+
   constructor(private _issueParams: IssueParams) {}
 
   self: GitHubIssue | null = null;
@@ -28,11 +35,32 @@ export class IssueActivity {
   linkedReviews: Review[] = [];
 
   async init() {
+    function fn<T>(func: () => Promise<T>) {
+      return func();
+    }
+    const decoratedFn = retryAsyncUntilDefinedDecorator(fn, {
+      delay: this._configuration.delayMs,
+      maxTry: this._configuration.maxAttempts,
+      async onError(error) {
+        try {
+          const content = "Failed to retrieve activity. Retrying...";
+          const message = logger.error(content, { error });
+          await githubCommentModuleInstance.postComment(message?.logMessage.diff || content);
+        } catch (e) {
+          logger.error(`${e}`);
+        }
+      },
+      async onMaxRetryFunc(error) {
+        logger.error("Failed to retrieve activity after 10 attempts. See logs for more details.", {
+          error,
+        });
+      },
+    });
     [this.self, this.events, this.comments, this.linkedReviews] = await Promise.all([
-      getIssue(this._issueParams),
-      getIssueEvents(this._issueParams),
-      getIssueComments(this._issueParams),
-      this._getLinkedReviews(),
+      decoratedFn(() => getIssue(this._issueParams)),
+      decoratedFn(() => getIssueEvents(this._issueParams)),
+      decoratedFn(() => getIssueComments(this._issueParams)),
+      decoratedFn(() => this._getLinkedReviews()),
     ]);
   }
 

tests/__mocks__/results/valid-configuration.json

@@ -2,6 +2,10 @@
   "evmNetworkId": 100,
   "evmPrivateEncrypted": "kmpTKq5Wh9r9x5j3U9GqZr3NYnjK2g0HtbzeUBOuLC2y3x8ja_SKBNlB2AZ6LigXHP_HeMitftVUtzmoj8CFfVP9SqjWoL6IPku1hVTWkdTn97g1IxzmjydFxjdcf0wuDW1hvVtoq3Uw5yALABqxcQ",
   "erc20RewardToken": "0xe91D153E0b41518A2Ce8Dd3D7944Fa863463a97d",
+  "dataCollection": {
+    "delayMs": 10000,
+    "maxAttempts": 10
+  },
   "incentives": {
     "enabled": true,
     "requirePriceLabel": true,

tests/action.test.ts

@@ -1,6 +1,9 @@
 /* eslint @typescript-eslint/no-var-requires: 0 */
 import "../src/parser/command-line";
+import { http, HttpResponse } from "msw";
+import { IssueActivity } from "../src/issue-activity";
 import { run } from "../src/run";
+import { parseGitHubUrl } from "../src/start";
 import { server } from "./__mocks__/node";
 
 beforeAll(() => server.listen());
@@ -71,4 +74,25 @@ https://github.com/ubiquibot/conversation-rewards/actions/runs/1
 }
 -->`);
   });
+
+  it("Should retry to fetch on network failure", async () => {
+    // Fakes one crash per route retrieving the data. Should succeed on retry. Timeout for the test function needs
+    // to be increased since it takes 10 seconds for a retry to happen.
+    [
+      "https://api.github.com/repos/ubiquibot/comment-incentives/issues/22",
+      "https://api.github.com/repos/ubiquibot/comment-incentives/issues/22/events",
+      "https://api.github.com/repos/ubiquibot/comment-incentives/issues/22/comments",
+      "https://api.github.com/repos/ubiquibot/comment-incentives/issues/22/timeline",
+    ].forEach((url) => {
+      server.use(http.get(url, () => HttpResponse.json("", { status: 500 }), { once: true }));
+    });
+    const issueUrl = process.env.TEST_ISSUE_URL || "https://github.com/ubiquibot/comment-incentives/issues/22";
+    const issue = parseGitHubUrl(issueUrl);
+    const activity = new IssueActivity(issue);
+    await activity.init();
+    expect(activity.self).toBeTruthy();
+    expect(activity.linkedReviews.length).toBeGreaterThan(0);
+    expect(activity.comments.length).toBeGreaterThan(0);
+    expect(activity.events.length).toBeGreaterThan(0);
+  }, 60000);
 });

yarn.lock

@@ -8334,6 +8334,11 @@ [email protected]:
     v8-compile-cache-lib "^3.0.1"
     yn "3.1.1"
 
+[email protected]:
+  version "4.2.5"
+  resolved "https://registry.yarnpkg.com/ts-retry/-/ts-retry-4.2.5.tgz#ee4638e66c68bb49da975aa4994d5f16bfb61bc2"
+  integrity sha512-dFBa4pxMBkt/bjzdBio8EwYfbAdycEAwe0KZgzlUKKwU9Wr1WErK7Hg9QLqJuDDYJXTW4KYZyXAyqYKOdO/ehA==
+
 [email protected]:
   version "2.4.0"
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.0.tgz#7cecaa7f073ce680a05847aa77be941098f36dc3"