📦 Add myst-execute package (#866)

Co-authored-by: Angus Hollands <[email protected]>
Co-authored-by: Rowan Cockett <[email protected]>

.changeset/khaki-tomatoes-remain.md

@@ -0,0 +1,5 @@
+---
+'myst-execute': patch
+---
+
+Add myst-execute package

.changeset/thick-cups-hear.md

@@ -0,0 +1,6 @@
+---
+'myst-common': minor
+'myst-cli': minor
+---
+
+Move IExpressionResult types to myst-common

.github/requirements/requirements-execute.txt

@@ -0,0 +1,2 @@
+jupyter-server
+ipykernel

.github/workflows/ci.yml

@@ -40,5 +40,12 @@ jobs:
         with:
           node-version: ${{ matrix.node }}
           cache: 'npm'
+      # Setup Python for ipykernel & jupyter-server
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+          cache-dependency-path: '.github/requirements/requirements-execute.txt'
+      - run: python -m pip install -r .github/requirements/requirements-execute.txt
       - run: npm install
       - run: npm run test

packages/myst-cli/src/transforms/inlineExpressions.ts

@@ -1,4 +1,4 @@
-import type { GenericNode, GenericParent } from 'myst-common';
+import type { GenericNode, GenericParent, IExpressionResult } from 'myst-common';
 import { fileWarn, NotebookCell, RuleId } from 'myst-common';
 import { selectAll } from 'unist-util-select';
 import type { InlineExpression } from 'myst-spec-ext';
@@ -10,25 +10,6 @@ import { BASE64_HEADER_SPLIT } from './images.js';
 
 export const metadataSection = 'user_expressions';
 
-export interface IBaseExpressionResult {
-  status: string;
-}
-
-export interface IExpressionOutput extends IBaseExpressionResult {
-  status: 'ok';
-  data: Record<string, string>;
-  metadata: Record<string, string>;
-}
-
-export interface IExpressionError extends IBaseExpressionResult {
-  status: 'error';
-  traceback: string[];
-  ename: string;
-  evalue: string;
-}
-
-export type IExpressionResult = IExpressionError | IExpressionOutput;
-
 export interface IUserExpressionMetadata {
   expression: string;
   result: IExpressionResult;
@@ -69,11 +50,11 @@ function renderExpression(node: InlineExpression, file: VFile): StaticPhrasingCo
           },
         ];
       } else if (mimeType === 'text/latex') {
-        content = [{ type: 'inlineMath', value: processLatex(value) }];
+        content = [{ type: 'inlineMath', value: processLatex(value as string) }];
       } else if (mimeType === 'text/html') {
-        content = [{ type: 'html', value }];
+        content = [{ type: 'html', value: value as string }];
       } else if (mimeType === 'text/plain') {
-        content = [{ type: 'text', value }];
+        content = [{ type: 'text', value: value as string }];
       }
     });
     if (content) return content;

packages/myst-common/package.json

@@ -30,7 +30,9 @@
     "vfile-message": "^3.0.0"
   },
   "devDependencies": {
-    "myst-spec-ext": "^1.1.22",
-    "unist-builder": "3.0.0"
+    "myst-spec-ext": "^1.1.21",
+    "unist-builder": "3.0.0",
+    "@jupyterlab/nbformat": "^3.5.2",
+    "@lumino/coreutils": "^2.0.0"
   }
 }

packages/myst-common/src/index.ts

@@ -24,6 +24,7 @@ export {
   ParseTypesEnum,
   TargetKind,
 } from './types.js';
+export type { IExpressionResult, IExpressionError, IExpressionOutput } from './types.js';
 
 export type { MessageInfo } from './utils.js';
 export type {

packages/myst-common/src/types.ts

@@ -1,6 +1,8 @@
 import type { Plugin } from 'unified';
-import type { Node, Directive, Role } from 'myst-spec';
+import type { Directive, Node, Role } from 'myst-spec';
 import type { VFile } from 'vfile';
+import type * as nbformat from '@jupyterlab/nbformat';
+import type { PartialJSONObject } from '@lumino/coreutils';
 
 export type GenericNode<T extends Record<string, any> = Record<string, any>> = {
   type: string;
@@ -151,3 +153,27 @@ export enum AdmonitionKind {
   tip = 'tip',
   warning = 'warning',
 }
+
+export interface IExpressionOutput {
+  status: 'ok';
+  data: nbformat.IMimeBundle;
+  metadata: PartialJSONObject;
+}
+
+export interface IExpressionError {
+  status: 'error';
+  /**
+   * Exception name
+   */
+  ename: string;
+  /**
+   * Exception value
+   */
+  evalue: string;
+  /**
+   * Traceback
+   */
+  traceback: string[];
+}
+
+export type IExpressionResult = IExpressionError | IExpressionOutput;

packages/myst-execute/.eslintrc.cjs

@@ -0,0 +1,4 @@
+module.exports = {
+  root: true,
+  extends: ['curvenote'],
+};

packages/myst-execute/.gitignore

@@ -0,0 +1 @@
+./tests/execute

packages/myst-execute/README.md

@@ -0,0 +1,5 @@
+# MyST Execute
+
+Kernel execution of code and inline expressions using Jupyter.
+
+This package provides a transform that takes a MyST tree, computes the executable content, and stores the results in the AST. It provides basic support for caching of outputs to avoid needless computation.

packages/myst-execute/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "myst-execute",
+  "version": "0.0.0",
+  "sideEffects": false,
+  "license": "MIT",
+  "description": "Execute MyST content with Jupyter",
+  "author": "Franklin Koch <[email protected]>",
+  "homepage": "https://github.com/executablebooks/mystmd/tree/main/packages/myst-execution",
+  "type": "module",
+  "exports": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/executablebooks/mystmd.git"
+  },
+  "scripts": {
+    "clean": "rimraf dist",
+    "lint": "eslint \"src/**/!(*.spec).ts\" -c ./.eslintrc.cjs",
+    "lint:format": "npx prettier --check \"src/**/*.ts\"",
+    "test": "vitest run",
+    "test:watch": "vitest watch",
+    "build:esm": "tsc",
+    "build": "npm-run-all -l clean -p build:esm"
+  },
+  "bugs": {
+    "url": "https://github.com/executablebooks/mystmd/issues"
+  },
+  "dependencies": {
+    "myst-common": "^1.1.21",
+    "vfile": "^5.3.7",
+    "@jupyterlab/services": "^7.0.0",
+    "unist-util-select": "^4.0.3",
+    "which": "^4.0.0"
+  },
+  "devDependencies": {
+    "js-yaml": "^4.1.0",
+    "node-fetch": "^3.3.0",
+    "@jupyterlab/nbformat": "^3.5.2"
+  }
+}

packages/myst-execute/src/cache.ts

@@ -0,0 +1,46 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+
+export interface ICache<T> {
+  test(key: string): boolean;
+
+  get(key: string): T | undefined;
+
+  set(key: string, value: T): void;
+}
+
+/**
+ * An implementation of a basic cache
+ */
+export class LocalDiskCache<T> implements ICache<T> {
+  constructor(cachePath: string) {
+    this._cachePath = cachePath;
+
+    if (!existsSync(cachePath)) {
+      mkdirSync(cachePath, { recursive: true });
+    }
+  }
+
+  private readonly _cachePath: string;
+
+  private _makeKeyPath(key: string): string {
+    return path.join(this._cachePath, `${key}.json`);
+  }
+
+  test(key: string): boolean {
+    return existsSync(this._makeKeyPath(key));
+  }
+
+  get(key: string): T | undefined {
+    const keyPath = this._makeKeyPath(key);
+    if (!existsSync(keyPath)) {
+      return undefined;
+    }
+    return JSON.parse(readFileSync(keyPath, { encoding: 'utf8' }));
+  }
+
+  set(key: string, item: T) {
+    const keyPath = this._makeKeyPath(key);
+    return writeFileSync(keyPath, JSON.stringify(item), { encoding: 'utf8' });
+  }
+}