Use client side rate limiting for plugin download

dev-packages/cli/package.json

@@ -44,6 +44,7 @@
     "chalk": "4.0.0",
     "decompress": "^4.2.1",
     "glob": "^8.0.3",
+    "limiter": "^2.1.0",
     "log-update": "^4.0.0",
     "mocha": "^10.1.0",
     "puppeteer": "^2.0.0",

dev-packages/cli/src/download-plugins.ts

@@ -32,6 +32,7 @@ import * as temp from 'temp';
 import { NodeRequestService } from '@theia/request/lib/node-request-service';
 import { DEFAULT_SUPPORTED_API_VERSION } from '@theia/application-package/lib/api';
 import { RequestContext } from '@theia/request';
+import { RateLimiter } from 'limiter';
 
 temp.track();
 
@@ -67,6 +68,8 @@ export interface DownloadPluginsOptions {
      */
     parallel?: boolean;
 
+    rateLimit?: number;
+
     proxyUrl?: string;
     proxyAuthorization?: string;
     strictSsl?: boolean;
@@ -86,7 +89,8 @@ export default async function downloadPlugins(options: DownloadPluginsOptions =
         ignoreErrors = false,
         apiVersion = DEFAULT_SUPPORTED_API_VERSION,
         apiUrl = 'https://open-vsx.org/api',
-        parallel = false,
+        parallel = true,
+        rateLimit = 15,
         proxyUrl,
         proxyAuthorization,
         strictSsl
@@ -120,8 +124,10 @@ export default async function downloadPlugins(options: DownloadPluginsOptions =
         }
     };
 
+    const rateLimiter = new RateLimiter({ tokensPerInterval: rateLimit, interval: 'second' });
+
     // Downloader wrapper
-    const downloadPlugin = (plugin: PluginDownload): Promise<void> => downloadPluginAsync(failures, plugin.id, plugin.downloadUrl, pluginsDir, packed, plugin.version);
+    const downloadPlugin = (plugin: PluginDownload): Promise<void> => downloadPluginAsync(rateLimiter, failures, plugin.id, plugin.downloadUrl, pluginsDir, packed, plugin.version);
 
     const downloader = async (plugins: PluginDownload[]) => {
         await parallelOrSequence(...plugins.map(plugin => () => downloadPlugin(plugin)));
@@ -148,10 +154,12 @@ export default async function downloadPlugins(options: DownloadPluginsOptions =
             const ids = new Set<string>(dependencies.flat());
             await parallelOrSequence(...Array.from(ids, id => async () => {
                 try {
+                    await rateLimiter.removeTokens(1);
                     const extension = await client.getLatestCompatibleExtensionVersion(id);
                     const version = extension?.version;
                     const downloadUrl = extension?.files.download;
                     if (downloadUrl) {
+                        await rateLimiter.removeTokens(1);
                         await downloadPlugin({ id, downloadUrl, version });
                     } else {
                         failures.push(`No download url for extension pack ${id} (${version})`);
@@ -196,7 +204,15 @@ export default async function downloadPlugins(options: DownloadPluginsOptions =
  * @param packed whether to decompress or not.
  * @param cachedExtensionPacks the list of cached extension packs already downloaded.
  */
-async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl: string, pluginsDir: string, packed: boolean, version?: string): Promise<void> {
+async function downloadPluginAsync(
+    rateLimiter: RateLimiter,
+    failures: string[],
+    plugin: string,
+    pluginUrl: string,
+    pluginsDir: string,
+    packed: boolean,
+    version?: string
+): Promise<void> {
     if (!plugin) {
         return;
     }
@@ -232,6 +248,7 @@ async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl
         }
         lastError = undefined;
         try {
+            await rateLimiter.removeTokens(1);
             response = await requestService.request({
                 url: pluginUrl
             });
@@ -240,7 +257,7 @@ async function downloadPluginAsync(failures: string[], plugin: string, pluginUrl
             continue;
         }
         const status = response.res.statusCode;
-        const retry = status && (status === 439 || status >= 500);
+        const retry = status && (status === 429 || status === 439 || status >= 500);
         if (!retry) {
             break;
         }

dev-packages/cli/src/theia.ts

@@ -343,7 +343,12 @@ async function theiaCli(): Promise<void> {
                 'parallel': {
                     describe: 'Download in parallel',
                     boolean: true,
-                    default: false
+                    default: true
+                },
+                'rate-limit': {
+                    describe: 'Amount of maximum open-vsx requests per second',
+                    number: true,
+                    default: 15
                 },
                 'proxy-url': {
                     describe: 'Proxy URL'

yarn.lock

@@ -7485,6 +7485,11 @@ just-extend@^4.0.2:
   resolved "https://registry.yarnpkg.com/just-extend/-/just-extend-4.2.1.tgz#ef5e589afb61e5d66b24eca749409a8939a8c744"
   integrity sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==
 
+[email protected]:
+  version "4.3.0"
+  resolved "https://registry.yarnpkg.com/just-performance/-/just-performance-4.3.0.tgz#cc2bc8c9227f09e97b6b1df4cd0de2df7ae16db1"
+  integrity sha512-L7RjvtJsL0QO8xFs5wEoDDzzJwoiowRw6Rn/GnvldlchS2JQr9wFYPiwZcDfrbbujEKqKN0tvENdbjXdYhDp5Q==
+
 [email protected]:
   version "7.2.0"
   resolved "https://registry.yarnpkg.com/keytar/-/keytar-7.2.0.tgz#4db2bec4f9700743ffd9eda22eebb658965c8440"
@@ -7599,6 +7604,13 @@ libnpmpublish@^6.0.4:
     semver "^7.3.7"
     ssri "^9.0.0"
 
+limiter@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/limiter/-/limiter-2.1.0.tgz#d38d7c5b63729bb84fb0c4d8594b7e955a5182a2"
+  integrity sha512-361TYz6iay6n+9KvUUImqdLuFigK+K79qrUtBsXhJTLdH4rIt/r1y8r1iozwh8KbZNpujbFTSh74mJ7bwbAMOw==
+  dependencies:
+    just-performance "4.3.0"
+
 line-height@^0.3.1:
   version "0.3.1"
   resolved "https://registry.yarnpkg.com/line-height/-/line-height-0.3.1.tgz#4b1205edde182872a5efa3c8f620b3187a9c54c9"