From f20b474769f0f49efd4e2a030f7cd188dc47e017 Mon Sep 17 00:00:00 2001 From: Mark Sujew Date: Tue, 6 Dec 2022 15:39:51 +0000 Subject: [PATCH] Use client side rate limiting for plugin download --- dev-packages/cli/package.json | 1 + dev-packages/cli/src/download-plugins.ts | 25 ++++++++++++++++++++---- dev-packages/cli/src/theia.ts | 7 ++++++- yarn.lock | 12 ++++++++++++ 4 files changed, 40 insertions(+), 5 deletions(-) diff --git a/dev-packages/cli/package.json b/dev-packages/cli/package.json index 5601e4d634ce4..b208e7e4d83b9 100644 --- a/dev-packages/cli/package.json +++ b/dev-packages/cli/package.json @@ -44,6 +44,7 @@ "chalk": "4.0.0", "decompress": "^4.2.1", "glob": "^8.0.3", + "limiter": "^2.1.0", "log-update": "^4.0.0", "mocha": "^10.1.0", "puppeteer": "^2.0.0", diff --git a/dev-packages/cli/src/download-plugins.ts b/dev-packages/cli/src/download-plugins.ts index 4f58970d344cc..95f54f5c150b2 100644 --- a/dev-packages/cli/src/download-plugins.ts +++ b/dev-packages/cli/src/download-plugins.ts @@ -32,6 +32,7 @@ import * as temp from 'temp'; import { NodeRequestService } from '@theia/request/lib/node-request-service'; import { DEFAULT_SUPPORTED_API_VERSION } from '@theia/application-package/lib/api'; import { RequestContext } from '@theia/request'; +import { RateLimiter } from 'limiter'; temp.track(); @@ -67,6 +68,8 @@ export interface DownloadPluginsOptions { */ parallel?: boolean; + rateLimit?: number; + proxyUrl?: string; proxyAuthorization?: string; strictSsl?: boolean; @@ -86,7 +89,8 @@ export default async function downloadPlugins(options: DownloadPluginsOptions = ignoreErrors = false, apiVersion = DEFAULT_SUPPORTED_API_VERSION, apiUrl = 'https://open-vsx.org/api', - parallel = false, + parallel = true, + rateLimit = 15, proxyUrl, proxyAuthorization, strictSsl @@ -120,8 +124,10 @@ export default async function downloadPlugins(options: DownloadPluginsOptions = } }; + const rateLimiter = new RateLimiter({ tokensPerInterval: rateLimit, interval: 'second' }); + // Downloader wrapper - const downloadPlugin = (plugin: PluginDownload): Promise => downloadPluginAsync(failures, plugin.id, plugin.downloadUrl, pluginsDir, packed, plugin.version); + const downloadPlugin = (plugin: PluginDownload): Promise => downloadPluginAsync(rateLimiter, failures, plugin.id, plugin.downloadUrl, pluginsDir, packed, plugin.version); const downloader = async (plugins: PluginDownload[]) => { await parallelOrSequence(...plugins.map(plugin => () => downloadPlugin(plugin))); @@ -148,10 +154,12 @@ export default async function downloadPlugins(options: DownloadPluginsOptions = const ids = new Set(dependencies.flat()); await parallelOrSequence(...Array.from(ids, id => async () => { try { + await rateLimiter.removeTokens(1); const extension = await client.getLatestCompatibleExtensionVersion(id); const version = extension?.version; const downloadUrl = extension?.files.download; if (downloadUrl) { + await rateLimiter.removeTokens(1); await downloadPlugin({ id, downloadUrl, version }); } else { failures.push(`No download url for extension pack ${id} (${version})`); @@ -196,7 +204,15 @@ export default async function downloadPlugins(options: DownloadPluginsOptions = * @param packed whether to decompress or not. * @param cachedExtensionPacks the list of cached extension packs already downloaded. */ -async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl: string, pluginsDir: string, packed: boolean, version?: string): Promise { +async function downloadPluginAsync( + rateLimiter: RateLimiter, + failures: string[], + plugin: string, + pluginUrl: string, + pluginsDir: string, + packed: boolean, + version?: string +): Promise { if (!plugin) { return; } @@ -232,6 +248,7 @@ async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl } lastError = undefined; try { + await rateLimiter.removeTokens(1); response = await requestService.request({ url: pluginUrl }); @@ -240,7 +257,7 @@ async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl continue; } const status = response.res.statusCode; - const retry = status && (status === 439 || status >= 500); + const retry = status && (status === 429 || status === 439 || status >= 500); if (!retry) { break; } diff --git a/dev-packages/cli/src/theia.ts b/dev-packages/cli/src/theia.ts index 24b6b12c01cae..3a5df9dc57848 100644 --- a/dev-packages/cli/src/theia.ts +++ b/dev-packages/cli/src/theia.ts @@ -343,7 +343,12 @@ async function theiaCli(): Promise { 'parallel': { describe: 'Download in parallel', boolean: true, - default: false + default: true + }, + 'rate-limit': { + describe: 'Amount of maximum open-vsx requests per second', + number: true, + default: 15 }, 'proxy-url': { describe: 'Proxy URL' diff --git a/yarn.lock b/yarn.lock index 9c4a14941c724..2e911c2cf1626 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7485,6 +7485,11 @@ just-extend@^4.0.2: resolved "https://registry.yarnpkg.com/just-extend/-/just-extend-4.2.1.tgz#ef5e589afb61e5d66b24eca749409a8939a8c744" integrity sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg== +just-performance@4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/just-performance/-/just-performance-4.3.0.tgz#cc2bc8c9227f09e97b6b1df4cd0de2df7ae16db1" + integrity sha512-L7RjvtJsL0QO8xFs5wEoDDzzJwoiowRw6Rn/GnvldlchS2JQr9wFYPiwZcDfrbbujEKqKN0tvENdbjXdYhDp5Q== + keytar@7.2.0: version "7.2.0" resolved "https://registry.yarnpkg.com/keytar/-/keytar-7.2.0.tgz#4db2bec4f9700743ffd9eda22eebb658965c8440" @@ -7599,6 +7604,13 @@ libnpmpublish@^6.0.4: semver "^7.3.7" ssri "^9.0.0" +limiter@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/limiter/-/limiter-2.1.0.tgz#d38d7c5b63729bb84fb0c4d8594b7e955a5182a2" + integrity sha512-361TYz6iay6n+9KvUUImqdLuFigK+K79qrUtBsXhJTLdH4rIt/r1y8r1iozwh8KbZNpujbFTSh74mJ7bwbAMOw== + dependencies: + just-performance "4.3.0" + line-height@^0.3.1: version "0.3.1" resolved "https://registry.yarnpkg.com/line-height/-/line-height-0.3.1.tgz#4b1205edde182872a5efa3c8f620b3187a9c54c9"