From a89fffd130e566803281a7a519efc7fc59aa5d99 Mon Sep 17 00:00:00 2001 From: gentlementlegen Date: Fri, 19 Jul 2024 11:23:26 +0900 Subject: [PATCH 1/6] feat: retry to fetch on error --- package.json | 1 + src/issue-activity.ts | 42 ++++++++++++++++++++++++++++++++++++------ yarn.lock | 5 +++++ 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 23df28f9..9a011776 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "lodash": "4.17.21", "markdown-it": "14.1.0", "openai": "4.29.1", + "ts-retry": "4.2.5", "tsx": "4.7.1", "typebox-validators": "0.3.5", "yaml": "2.4.1" diff --git a/src/issue-activity.ts b/src/issue-activity.ts index a149ef7c..5a47a53c 100644 --- a/src/issue-activity.ts +++ b/src/issue-activity.ts @@ -9,15 +9,18 @@ import { GitHubPullRequestReviewState, } from "./github-types"; import { - IssueParams, - PullParams, getIssue, getIssueComments, getIssueEvents, getPullRequest, getPullRequestReviewComments, getPullRequestReviews, + IssueParams, + PullParams, } from "./start"; +import { retryAsyncUntilDefinedDecorator } from "ts-retry"; +import githubCommentModuleInstance from "./helpers/github-comment-module-instance"; +import logger from "./helpers/logger"; export class IssueActivity { constructor(private _issueParams: IssueParams) {} @@ -28,11 +31,38 @@ export class IssueActivity { linkedReviews: Review[] = []; async init() { + function fn(func: () => Promise) { + return func(); + } + const decoratedFn = retryAsyncUntilDefinedDecorator(fn, { + delay: 10000, + maxTry: 10, + async onError(error) { + try { + const content = "Failed to retrieve activity. Retrying..."; + const message = logger.error(content, { error }); + await githubCommentModuleInstance.postComment(message?.logMessage.diff || content); + } catch (e) { + logger.error(`${e}`); + } + }, + async onMaxRetryFunc(error) { + try { + const content = "Failed to retrieve activity after 10 attempts. See logs for more details."; + const message = logger.error(content, { + error, + }); + await githubCommentModuleInstance.postComment(message?.logMessage.diff || content); + } catch (e) { + logger.error(`${e}`); + } + }, + }); [this.self, this.events, this.comments, this.linkedReviews] = await Promise.all([ - getIssue(this._issueParams), - getIssueEvents(this._issueParams), - getIssueComments(this._issueParams), - this._getLinkedReviews(), + decoratedFn(() => getIssue(this._issueParams)), + decoratedFn(() => getIssueEvents(this._issueParams)), + decoratedFn(() => getIssueComments(this._issueParams)), + decoratedFn(() => this._getLinkedReviews()), ]); } diff --git a/yarn.lock b/yarn.lock index 1b350d97..d066987e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8334,6 +8334,11 @@ ts-node@10.9.2: v8-compile-cache-lib "^3.0.1" yn "3.1.1" +ts-retry@4.2.5: + version "4.2.5" + resolved "https://registry.yarnpkg.com/ts-retry/-/ts-retry-4.2.5.tgz#ee4638e66c68bb49da975aa4994d5f16bfb61bc2" + integrity sha512-dFBa4pxMBkt/bjzdBio8EwYfbAdycEAwe0KZgzlUKKwU9Wr1WErK7Hg9QLqJuDDYJXTW4KYZyXAyqYKOdO/ehA== + tslib@2.4.0: version "2.4.0" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.0.tgz#7cecaa7f073ce680a05847aa77be941098f36dc3" From 82d75fe12c45d08cff4b59933158102070c2ec80 Mon Sep 17 00:00:00 2001 From: gentlementlegen Date: Fri, 19 Jul 2024 12:31:02 +0900 Subject: [PATCH 2/6] chore: removed post on failed attempts --- src/issue-activity.ts | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/issue-activity.ts b/src/issue-activity.ts index 5a47a53c..fed7ce3e 100644 --- a/src/issue-activity.ts +++ b/src/issue-activity.ts @@ -47,15 +47,9 @@ export class IssueActivity { } }, async onMaxRetryFunc(error) { - try { - const content = "Failed to retrieve activity after 10 attempts. See logs for more details."; - const message = logger.error(content, { - error, - }); - await githubCommentModuleInstance.postComment(message?.logMessage.diff || content); - } catch (e) { - logger.error(`${e}`); - } + logger.error("Failed to retrieve activity after 10 attempts. See logs for more details.", { + error, + }); }, }); [this.self, this.events, this.comments, this.linkedReviews] = await Promise.all([ From 26a0bed151188b95d901e79a58e874ab4c5280db Mon Sep 17 00:00:00 2001 From: Mentlegen <9807008+gentlementlegen@users.noreply.github.com> Date: Fri, 19 Jul 2024 13:26:13 +0900 Subject: [PATCH 3/6] feat: configuration for data collection --- README.md | 3 +++ src/configuration/data-collection-config.ts | 14 ++++++++++++++ src/configuration/incentives.ts | 3 +++ src/issue-activity.ts | 14 +++++++++----- 4 files changed, 29 insertions(+), 5 deletions(-) create mode 100644 src/configuration/data-collection-config.ts diff --git a/README.md b/README.md index 743b0dff..7a591562 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,9 @@ with: evmNetworkId: 100 evmPrivateEncrypted: "encrypted-key" erc20RewardToken: "0xe91D153E0b41518A2Ce8Dd3D7944Fa863463a97d" + dataCollection: + maxTry: 10 + delay: 10000 incentives: enabled: true requirePriceLabel: true diff --git a/src/configuration/data-collection-config.ts b/src/configuration/data-collection-config.ts new file mode 100644 index 00000000..67b20236 --- /dev/null +++ b/src/configuration/data-collection-config.ts @@ -0,0 +1,14 @@ +import { Static, Type } from "@sinclair/typebox"; + +export const dataCollectionConfigurationType = Type.Object({ + /** + * The maximum amount of retries on failure. + */ + maxTry: Type.Number({ default: 10, minimum: 1 }), + /** + * The delay between each retry, in milliseconds. + */ + delay: Type.Number({ default: 10000, minimum: 100 }), +}); + +export type DataCollectionConfiguration = Static; diff --git a/src/configuration/incentives.ts b/src/configuration/incentives.ts index 122dbcc1..cb2dd070 100644 --- a/src/configuration/incentives.ts +++ b/src/configuration/incentives.ts @@ -1,6 +1,7 @@ import { StaticDecode, Type as T } from "@sinclair/typebox"; import { StandardValidator } from "typebox-validators"; import { contentEvaluatorConfigurationType } from "./content-evaluator-config"; +import { dataCollectionConfigurationType } from "./data-collection-config"; import { dataPurgeConfigurationType } from "./data-purge-config"; import { formattingEvaluatorConfigurationType } from "./formatting-evaluator-config"; import { githubCommentConfigurationType } from "./github-comment-config"; @@ -40,7 +41,9 @@ export const incentivesConfigurationSchema = T.Object({ permitGeneration: permitGenerationConfigurationType, githubComment: githubCommentConfigurationType, }), + dataCollection: dataCollectionConfigurationType, }); + export const validateIncentivesConfiguration = new StandardValidator(incentivesConfigurationSchema); export type IncentivesConfiguration = StaticDecode; diff --git a/src/issue-activity.ts b/src/issue-activity.ts index fed7ce3e..2c5ee8d5 100644 --- a/src/issue-activity.ts +++ b/src/issue-activity.ts @@ -1,4 +1,7 @@ +import { retryAsyncUntilDefinedDecorator } from "ts-retry"; import { CommentType } from "./configuration/comment-types"; +import configuration from "./configuration/config-reader"; +import { DataCollectionConfiguration } from "./configuration/data-collection-config"; import { collectLinkedMergedPulls } from "./data-collection/collect-linked-pulls"; import { GitHubIssue, @@ -8,6 +11,8 @@ import { GitHubPullRequestReviewComment, GitHubPullRequestReviewState, } from "./github-types"; +import githubCommentModuleInstance from "./helpers/github-comment-module-instance"; +import logger from "./helpers/logger"; import { getIssue, getIssueComments, @@ -18,11 +23,10 @@ import { IssueParams, PullParams, } from "./start"; -import { retryAsyncUntilDefinedDecorator } from "ts-retry"; -import githubCommentModuleInstance from "./helpers/github-comment-module-instance"; -import logger from "./helpers/logger"; export class IssueActivity { + readonly _configuration: DataCollectionConfiguration = configuration.dataCollection; + constructor(private _issueParams: IssueParams) {} self: GitHubIssue | null = null; @@ -35,8 +39,8 @@ export class IssueActivity { return func(); } const decoratedFn = retryAsyncUntilDefinedDecorator(fn, { - delay: 10000, - maxTry: 10, + delay: this._configuration.delay, + maxTry: this._configuration.maxTry, async onError(error) { try { const content = "Failed to retrieve activity. Retrying..."; From 50f499e931c769e39c1321b463bf1207d27b512a Mon Sep 17 00:00:00 2001 From: Mentlegen <9807008+gentlementlegen@users.noreply.github.com> Date: Fri, 19 Jul 2024 13:56:16 +0900 Subject: [PATCH 4/6] chore: added tests for retry --- .../results/valid-configuration.json | 4 ++++ tests/action.test.ts | 22 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/tests/__mocks__/results/valid-configuration.json b/tests/__mocks__/results/valid-configuration.json index 43c9775c..3a0bfcbd 100644 --- a/tests/__mocks__/results/valid-configuration.json +++ b/tests/__mocks__/results/valid-configuration.json @@ -2,6 +2,10 @@ "evmNetworkId": 100, "evmPrivateEncrypted": "kmpTKq5Wh9r9x5j3U9GqZr3NYnjK2g0HtbzeUBOuLC2y3x8ja_SKBNlB2AZ6LigXHP_HeMitftVUtzmoj8CFfVP9SqjWoL6IPku1hVTWkdTn97g1IxzmjydFxjdcf0wuDW1hvVtoq3Uw5yALABqxcQ", "erc20RewardToken": "0xe91D153E0b41518A2Ce8Dd3D7944Fa863463a97d", + "dataCollection": { + "delay": 10000, + "maxTry": 10 + }, "incentives": { "enabled": true, "requirePriceLabel": true, diff --git a/tests/action.test.ts b/tests/action.test.ts index fa38d4f8..ca8eb271 100644 --- a/tests/action.test.ts +++ b/tests/action.test.ts @@ -1,6 +1,9 @@ /* eslint @typescript-eslint/no-var-requires: 0 */ import "../src/parser/command-line"; +import { http, HttpResponse } from "msw"; +import { IssueActivity } from "../src/issue-activity"; import { run } from "../src/run"; +import { parseGitHubUrl } from "../src/start"; import { server } from "./__mocks__/node"; beforeAll(() => server.listen()); @@ -71,4 +74,23 @@ https://github.com/ubiquibot/conversation-rewards/actions/runs/1 } -->`); }); + + it("Should retry to fetch on network failure", async () => { + // Fakes one crash per route retrieving the data. Should succeed on retry. Timeout for the test function needs + // to be increased since it takes 10 seconds for a retry to happen. + [ + "https://api.github.com/repos/ubiquibot/comment-incentives/issues/22", + "https://api.github.com/repos/ubiquibot/comment-incentives/issues/22/events", + "https://api.github.com/repos/ubiquibot/comment-incentives/issues/22/comments", + "https://api.github.com/repos/ubiquibot/comment-incentives/issues/22/timeline", + ].forEach((url) => { + server.use(http.get(url, () => HttpResponse.json("", { status: 500 }), { once: true })); + }); + const issueUrl = process.env.TEST_ISSUE_URL || "https://github.com/ubiquibot/comment-incentives/issues/22"; + const issue = parseGitHubUrl(issueUrl); + const activity = new IssueActivity(issue); + await activity.init(); + expect(activity.self).toBeTruthy(); + expect([]).toBeTruthy(); + }, 60000); }); From 91443682f06e04e58f88a2964dae257ef53e7b41 Mon Sep 17 00:00:00 2001 From: Mentlegen <9807008+gentlementlegen@users.noreply.github.com> Date: Fri, 19 Jul 2024 13:59:36 +0900 Subject: [PATCH 5/6] chore: added tests for retry --- tests/action.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/action.test.ts b/tests/action.test.ts index ca8eb271..3ce2e9c8 100644 --- a/tests/action.test.ts +++ b/tests/action.test.ts @@ -91,6 +91,8 @@ https://github.com/ubiquibot/conversation-rewards/actions/runs/1 const activity = new IssueActivity(issue); await activity.init(); expect(activity.self).toBeTruthy(); - expect([]).toBeTruthy(); + expect(activity.linkedReviews.length).toBeGreaterThan(0); + expect(activity.comments.length).toBeGreaterThan(0); + expect(activity.events.length).toBeGreaterThan(0); }, 60000); }); From 69cb9b58413a87e41557907d797e212b367f68de Mon Sep 17 00:00:00 2001 From: Mentlegen <9807008+gentlementlegen@users.noreply.github.com> Date: Fri, 19 Jul 2024 14:55:03 +0900 Subject: [PATCH 6/6] chore: rename maxTry and delay --- README.md | 4 ++-- src/configuration/data-collection-config.ts | 4 ++-- src/issue-activity.ts | 4 ++-- tests/__mocks__/results/valid-configuration.json | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 7a591562..d9cb2b22 100644 --- a/README.md +++ b/README.md @@ -56,8 +56,8 @@ with: evmPrivateEncrypted: "encrypted-key" erc20RewardToken: "0xe91D153E0b41518A2Ce8Dd3D7944Fa863463a97d" dataCollection: - maxTry: 10 - delay: 10000 + maxAttempts: 10 + delayMs: 10000 incentives: enabled: true requirePriceLabel: true diff --git a/src/configuration/data-collection-config.ts b/src/configuration/data-collection-config.ts index 67b20236..59b4982f 100644 --- a/src/configuration/data-collection-config.ts +++ b/src/configuration/data-collection-config.ts @@ -4,11 +4,11 @@ export const dataCollectionConfigurationType = Type.Object({ /** * The maximum amount of retries on failure. */ - maxTry: Type.Number({ default: 10, minimum: 1 }), + maxAttempts: Type.Number({ default: 10, minimum: 1 }), /** * The delay between each retry, in milliseconds. */ - delay: Type.Number({ default: 10000, minimum: 100 }), + delayMs: Type.Number({ default: 10000, minimum: 100 }), }); export type DataCollectionConfiguration = Static; diff --git a/src/issue-activity.ts b/src/issue-activity.ts index 2c5ee8d5..a2063d0d 100644 --- a/src/issue-activity.ts +++ b/src/issue-activity.ts @@ -39,8 +39,8 @@ export class IssueActivity { return func(); } const decoratedFn = retryAsyncUntilDefinedDecorator(fn, { - delay: this._configuration.delay, - maxTry: this._configuration.maxTry, + delay: this._configuration.delayMs, + maxTry: this._configuration.maxAttempts, async onError(error) { try { const content = "Failed to retrieve activity. Retrying..."; diff --git a/tests/__mocks__/results/valid-configuration.json b/tests/__mocks__/results/valid-configuration.json index 3a0bfcbd..8f4f69ff 100644 --- a/tests/__mocks__/results/valid-configuration.json +++ b/tests/__mocks__/results/valid-configuration.json @@ -3,8 +3,8 @@ "evmPrivateEncrypted": "kmpTKq5Wh9r9x5j3U9GqZr3NYnjK2g0HtbzeUBOuLC2y3x8ja_SKBNlB2AZ6LigXHP_HeMitftVUtzmoj8CFfVP9SqjWoL6IPku1hVTWkdTn97g1IxzmjydFxjdcf0wuDW1hvVtoq3Uw5yALABqxcQ", "erc20RewardToken": "0xe91D153E0b41518A2Ce8Dd3D7944Fa863463a97d", "dataCollection": { - "delay": 10000, - "maxTry": 10 + "delayMs": 10000, + "maxAttempts": 10 }, "incentives": { "enabled": true,