🌎 Allow https proxy from env var

.changeset/dull-falcons-obey.md

@@ -0,0 +1,9 @@
+---
+'myst-cli-utils': patch
+'myst-templates': patch
+'myst-execute': patch
+'myst-to-jats': patch
+'myst-cli': patch
+---
+
+Move fetch function to session

.changeset/famous-cats-brake.md

@@ -0,0 +1,5 @@
+---
+'myst-cli': patch
+---
+
+Allow https proxy set as environment var

docs/installing.md

@@ -49,3 +49,12 @@ Try the `myst --version` command before and after, with an update you should be
 ### Dependencies for $\LaTeX$ and PDF
 
 If you are exporting to $\LaTeX$ with an open-source template specified (see all [templates](https://github.com/myst-templates)) or if you are creating a PDF you will need to install a version of [LaTeX](https://www.latex-project.org/get).
+
+## Working with a Proxy Policy
+
+MyST performs web requests to download templates, check DOIs, etc. If you are working on a network that enforces a strict proxy policy for internet access, you may specify a proxy configuration string with the `HTTPS_PROXY` environment variable, for example:
+
+```shell
+HTTPS_PROXY=http://168.63.76.32:3128 \
+myst build
+```

packages/myst-cli-utils/package.json

@@ -36,6 +36,7 @@
     "pretty-hrtime": "^1.0.3"
   },
   "devDependencies": {
-    "@types/pretty-hrtime": "^1.0.1"
+    "@types/pretty-hrtime": "^1.0.1",
+    "node-fetch": "^3.3.0"
   }
 }

packages/myst-cli-utils/src/index.ts

@@ -10,7 +10,6 @@ export {
 export { exec, makeExecutable } from './exec.js';
 export { clirun, tic } from './utils.js';
 export { isUrl } from './isUrl.js';
-export { Session, getSession } from './session.js';
 export {
   computeHash,
   copyFileMaintainPath,

packages/myst-cli-utils/src/types.ts

@@ -1,7 +1,10 @@
+import type { RequestInfo, RequestInit, Response } from 'node-fetch';
+
 export type Logger = Pick<typeof console, 'debug' | 'info' | 'warn' | 'error'>;
 
 export type LoggerDE = Pick<Logger, 'debug' | 'error'>;
 
 export interface ISession {
   log: Logger;
+  fetch(url: URL | RequestInfo, init?: RequestInit): Promise<Response>;
 }

packages/myst-cli/package.json

@@ -58,6 +58,7 @@
     "fs-extra": "^11.1.1",
     "get-port": "^6.1.2",
     "glob": "^10.3.1",
+    "https-proxy-agent": "^7.0.4",
     "inquirer": "^8.2.2",
     "intersphinx": "^1.0.2",
     "js-yaml": "^4.1.0",

packages/myst-cli/src/build/html/index.ts

@@ -1,7 +1,6 @@
 import fs from 'fs-extra';
 import path from 'node:path';
 import { writeFileToFolder } from 'myst-cli-utils';
-import fetch from 'node-fetch';
 import type { ISession } from '../../session/types.js';
 import { getSiteManifest } from '../site/manifest.js';
 import { getMystTemplate } from '../site/template.js';
@@ -111,7 +110,7 @@ export async function buildHtml(session: ISession, opts: any) {
   // Fetch all HTML pages and assets by the template
   await Promise.all(
     routes.map(async (page) => {
-      const resp = await fetch(page.url);
+      const resp = await session.fetch(page.url);
       const content = await resp.text();
       writeFileToFolder(path.join(htmlDir, page.path), content);
     }),

packages/myst-cli/src/process/citations.ts

@@ -1,5 +1,4 @@
 import fs from 'node:fs';
-import fetch from 'node-fetch';
 import type { CitationRenderer } from 'citation-js-utils';
 import { getCitations } from 'citation-js-utils';
 import { tic, isUrl } from 'myst-cli-utils';
@@ -14,7 +13,7 @@ export async function loadCitations(session: ISession, path: string): Promise<Ci
   let data: string;
   if (isUrl(path)) {
     session.log.debug(`Fetching citations at "${path}"`);
-    const res = await fetch(path);
+    const res = await session.fetch(path);
     if (!res.ok) {
       throw new Error(`Error fetching citations from "${path}": ${res.status} ${res.statusText}`);
     }

packages/myst-cli/src/session/session.ts

@@ -7,6 +7,7 @@ import type { MystPlugin, RuleId } from 'myst-common';
 import latestVersion from 'latest-version';
 import boxen from 'boxen';
 import chalk from 'chalk';
+import { HttpsProxyAgent } from 'https-proxy-agent';
 import {
   findCurrentProjectAndLoad,
   findCurrentSiteAndLoad,
@@ -22,6 +23,7 @@ import type { ISession } from './types.js';
 import { KernelManager, ServerConnection, SessionManager } from '@jupyterlab/services';
 import type { JupyterServerSettings } from 'myst-execute';
 import { findExistingJupyterServer, launchJupyterServer } from 'myst-execute';
+import type { RequestInfo, RequestInit } from 'node-fetch';
 import { default as nodeFetch, Headers, Request, Response } from 'node-fetch';
 
 // fetch polyfill for node<18
@@ -36,6 +38,7 @@ const CONFIG_FILES = ['myst.yml'];
 const API_URL = 'https://api.mystmd.org';
 const NPM_COMMAND = 'npm i -g mystmd@latest';
 const PIP_COMMAND = 'pip install -U mystmd';
+const LOCALHOSTS = ['localhost', '127.0.0.1', '::1'];
 
 export function logUpdateAvailable({
   current,
@@ -72,6 +75,7 @@ export class Session implements ISession {
   store: Store<RootState>;
   $logger: Logger;
 
+  proxyAgent?: HttpsProxyAgent<string>;
   _shownUpgrade = false;
   _latestVersion?: string;
   _jupyterSessionManagerPromise?: Promise<SessionManager | undefined>;
@@ -84,6 +88,8 @@ export class Session implements ISession {
     this.API_URL = API_URL;
     this.configFiles = CONFIG_FILES;
     this.$logger = opts.logger ?? chalkLogger(LogLevel.info, process.cwd());
+    const proxyUrl = process.env.HTTPS_PROXY;
+    if (proxyUrl) this.proxyAgent = new HttpsProxyAgent(proxyUrl);
     this.store = createStore(rootReducer);
     this.reload();
     // Allow the latest version to be loaded
@@ -116,6 +122,18 @@ export class Session implements ISession {
     return this;
   }
 
+  async fetch(url: URL | RequestInfo, init?: RequestInit): Promise<Response> {
+    const urlOnly = new URL((url as Request).url ?? (url as URL | string));
+    this.log.debug(`Fetching: ${urlOnly}`);
+    if (this.proxyAgent && !LOCALHOSTS.includes(urlOnly.hostname)) {
+      if (!init) init = {};
+      init = { agent: this.proxyAgent, ...init };
+      this.log.debug(`Using HTTPS proxy: ${this.proxyAgent.proxy}`);
+    }
+    const resp = await nodeFetch(url, init);
+    return resp;
+  }
+
   plugins: MystPlugin | undefined;
 
   _pluginPromise: Promise<MystPlugin> | undefined;
@@ -196,7 +214,7 @@ export class Session implements ISession {
         };
       } else {
         // Load existing running server
-        const existing = await findExistingJupyterServer();
+        const existing = await findExistingJupyterServer(this);
         if (existing) {
           this.log.debug(`Found existing server on: ${existing.appUrl}`);
           partialServerSettings = existing;

packages/myst-cli/src/session/types.ts

@@ -5,7 +5,7 @@ import type { MystPlugin, RuleId } from 'myst-common';
 import type { ReferenceState } from 'myst-transforms';
 import type { MinifiedContentCache } from 'nbtx';
 import type { Store } from 'redux';
-
+import type { RequestInfo, RequestInit, Response } from 'node-fetch';
 import type { BuildWarning, RootState } from '../store/index.js';
 import type { PreRendererData, RendererData, SingleCitationRenderer } from '../transforms/types.js';
 import type { SessionManager } from '@jupyterlab/services';
@@ -27,6 +27,7 @@ export type ISession = {
   getAllWarnings(ruleId: RuleId): (BuildWarning & { file: string })[];
   jupyterSessionManager(): Promise<SessionManager | undefined>;
   dispose(): void;
+  fetch(url: URL | RequestInfo, init?: RequestInit): Promise<Response>;
 };
 
 export type ISessionWithCache = ISession & {

packages/myst-cli/src/transforms/dois.ts

@@ -7,7 +7,6 @@ import type { Link } from 'myst-spec';
 import type { GenericNode, GenericParent } from 'myst-common';
 import { fileWarn, toText, RuleId, plural } from 'myst-common';
 import { selectAll } from 'unist-util-select';
-import fetch from 'node-fetch';
 import { computeHash, tic } from 'myst-cli-utils';
 import type { Cite } from 'myst-spec-ext';
 import type { SingleCitationRenderer } from './types.js';
@@ -36,12 +35,14 @@ export async function getDoiOrgBibtex(
   const toc = tic();
   session.log.debug('Fetching DOI information from doi.org');
   const url = `https://doi.org/${normalizedDoi}`;
-  const response = await fetch(url, {
-    headers: [['Accept', 'application/x-bibtex']],
-  }).catch(() => {
-    session.log.debug(`Request to ${url} failed.`);
-    return null;
-  });
+  const response = await session
+    .fetch(url, {
+      headers: [['Accept', 'application/x-bibtex']],
+    })
+    .catch(() => {
+      session.log.debug(`Request to ${url} failed.`);
+      return null;
+    });
   if (!response || !response.ok) {
     session.log.debug(`doi.org fetch failed for ${doiString}}`);
     return null;

packages/myst-cli/src/transforms/images.ts

@@ -5,7 +5,6 @@ import { RuleId, plural } from 'myst-common';
 import { computeHash, hashAndCopyStaticFile, isUrl } from 'myst-cli-utils';
 import { remove } from 'unist-util-remove';
 import { selectAll } from 'unist-util-select';
-import fetch from 'node-fetch';
 import path from 'node:path';
 import type { VFileMessage } from 'vfile-message';
 import type { PageFrontmatter } from 'myst-frontmatter';
@@ -75,7 +74,7 @@ export async function downloadAndSaveImage(
   session.log.debug(`Fetching image: ${url}...\n  -> saving to: ${filePath}`);
   try {
     const github = getGithubRawUrl(url);
-    const res = await fetch(github ?? url, { headers: EXT_REQUEST_HEADERS });
+    const res = await session.fetch(github ?? url, { headers: EXT_REQUEST_HEADERS });
     const contentType = res.headers.get('content-type') || '';
     const extension = mime.extension(contentType);
     if (!extension || !contentType) throw new Error('No content-type for image found.');

packages/myst-cli/src/transforms/links.ts

@@ -1,7 +1,6 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import pLimit from 'p-limit';
-import fetch from 'node-fetch';
 import type { GenericNode, GenericParent } from 'myst-common';
 import { selectAll } from 'unist-util-select';
 import { updateLinkTextIfEmpty } from 'myst-transforms';
@@ -64,7 +63,7 @@ export async function checkLink(session: ISession, url: string): Promise<Externa
     }
     session.log.debug(`Checking that "${url}" exists`);
     const linkCache = loadLinkCache(session, url);
-    const resp = linkCache ?? (await fetch(url, { headers: EXT_REQUEST_HEADERS }));
+    const resp = linkCache ?? (await session.fetch(url, { headers: EXT_REQUEST_HEADERS }));
     link.ok = resp.ok;
     link.status = resp.status;
     link.statusText = resp.statusText;

packages/myst-execute/package.json

@@ -32,16 +32,17 @@
     "url": "https://github.com/executablebooks/mystmd/issues"
   },
   "dependencies": {
+    "@jupyterlab/services": "^7.0.0",
     "chalk": "^5.2.0",
+    "myst-cli-utils": "^2.0.7",
     "myst-common": "^1.1.24",
-    "vfile": "^5.3.7",
-    "@jupyterlab/services": "^7.0.0",
+    "node-fetch": "^3.3.0",
     "unist-util-select": "^4.0.3",
-    "which": "^4.0.0",
-    "node-fetch": "^3.3.0"
+    "vfile": "^5.3.7",
+    "which": "^4.0.0"
   },
   "devDependencies": {
-    "js-yaml": "^4.1.0",
-    "@jupyterlab/nbformat": "^3.5.2"
+    "@jupyterlab/nbformat": "^3.5.2",
+    "js-yaml": "^4.1.0"
   }
 }

packages/myst-execute/src/manager.ts

@@ -2,9 +2,8 @@ import type { ServerConnection } from '@jupyterlab/services';
 import which from 'which';
 import { spawn } from 'node:child_process';
 import * as readline from 'node:readline';
-import type { Logger } from 'myst-cli-utils';
+import type { ISession, Logger } from 'myst-cli-utils';
 import chalk from 'chalk';
-import fetch from 'node-fetch';
 
 export type JupyterServerSettings = Partial<ServerConnection.ISettings> & {
   dispose?: () => void;
@@ -27,7 +26,9 @@ interface JupyterServerListItem {
 /**
  * Find the newest (by PID) active Jupyter Server, or return undefined.
  */
-export async function findExistingJupyterServer(): Promise<JupyterServerSettings | undefined> {
+export async function findExistingJupyterServer(
+  session: ISession,
+): Promise<JupyterServerSettings | undefined> {
   const pythonPath = which.sync('python');
   const listProc = spawn(pythonPath, ['-m', 'jupyter_server', 'list', '--json']);
 
@@ -52,7 +53,7 @@ export async function findExistingJupyterServer(): Promise<JupyterServerSettings
 
   // Return the first alive server
   for (const entry of servers) {
-    const response = await fetch(`${entry.url}?token=${entry.token}`);
+    const response = await session.fetch(`${entry.url}?token=${entry.token}`);
     if (response.ok) {
       return {
         baseUrl: entry.url,