Add sample tool hosting app in ExpressJS (#239)

banking-app/rollup.config.js

@@ -2,6 +2,9 @@ import { nodeResolve } from "@rollup/plugin-node-resolve";
 import commonjs from "@rollup/plugin-commonjs";
 import typescript from "@rollup/plugin-typescript";
 
+/**
+ * @type {import('rollup').RollupOptions}
+ */
 export default {
   input: "src/endpoints/all.ts",
   output: {

express-debug/README.md

@@ -0,0 +1,73 @@
+# CCF Express
+
+## Summary
+This folder contains a small tool to host a CCF JS app using ExpressJS. This uses the polyfills distributed with the `ccf-app` npm package to implement the `ccf` global, and exposes HTTP routes based on the metadata from an `app.json` file.
+
+The intention is that this can be used to debug CCF application handlers written in JavaScript or TypeScript. This does not start a multi-node network, or any enclaved code. Instead it starts a single local web server, attempting to provide the same `/app` HTTP API defined by `app.json`.
+
+### *WARNINGS*
+This is a minimal PoC, and does not provide a perfectly matching server implementation. Notable discrepancies from a CCF node at the time of writing:
+- No implementation of governance endpoints. This purely serves `/app` endpoints, and any attempts at governance will return 404s. Any app state which is bootstrapped by governance may need additional genesis calls.
+- Does no authentication. It attempts to construct a valid `caller` object based on fields from the incoming request, but does not validate that this represents an identity that is known and trusted in the KV.
+- KV does not offer opacity or snapshot isolation. Any writes to the KV are immediately visible to other concurrently executing handlers, and writes are _not_ rolled back for failing operations.
+- Error response body shape is inconsistent. This does not produce JSON OData responses for errors in the same way a CCF node does.
+
+In short, this should only be used for testing happy-path execution flows. Any flows which rely on error responses or framework-inserted details are not supported out-of-the-box, and will require extensions to this tool.
+
+## Issues
+
+The approach used by this tool is to hook into the module loader (`--experimental-loader ./bundle_loader.js`) to resolve all imports directly from a single bundle file. This mirrors the CCF deployment path, ensuring all dependencies are atomically packaged, but is unnecessarily cumbersome, and leads to constant rewriting of files into a temporary `dist/` folder. Additionally it means the debugger is stepping into temporary files created within the `dist/` folder, rather than the original source files.
+
+It should instead be possible to import modules from their original locations on the local hard drive with standard import tooling, either with dynamic `import()` (as this does), or by generating the `main.js` application ahead-of-time.
+
+## Use
+
+The input file for this tool is a `bundle.json`, containing the routes exposed by a CCF app and all associated JS code. This object is the one emitted by the sample `build_bundle.js`, or contained in the `bundle` field of a `set_js_app` proposal.
+
+For instance, to run the `banking-app` sample contained in this repo:
+
+```bash
+$ cd banking-app
+
+$ npm run build
+
+$ cd ../express-debug
+
+$ node --experimental-loader ./bundle_loader.js ./main.js --bundle ../banking-app/dist/bundle.json
+Writing server certificate to ./service_cert.pem
+CCF express app listening on :::8000!
+```
+
+Note this emits a new self-signed `service_cert.pem` used as the server identity. Clients must either obtain this, or disable TLS server authentication (`-k` to curl).
+
+Interact with the server like any other CCF node or web server:
+
+```bash
+$ curl https://localhost:8000/app/balance/bob --cacert ./service_cert.pem
+```
+
+## VSCode integration
+
+To launch this tool with the VSCode debugger attached, add a VSCode launch configuration (to `.vscode/launch.json`) based on the following snippet:
+
+```json
+{
+  "configurations": [
+    {
+      "type": "node",
+      "request": "launch",
+      "cwd": "${workspaceFolder}/express-debug/",
+      "name": "Host current bundle with ExpressJS",
+      "skipFiles": ["<node_internals>/**"],
+      "args": ["--bundle", "${file}"],
+      "runtimeArgs": [
+        "--experimental-loader",
+        "${workspaceFolder}/express-debug/bundle_loader.js"
+      ],
+      "program": "${workspaceFolder}/express-debug/main.js"
+    }
+  ]
+}
+```
+
+To use this configuration, first select a `bundle.json` as the active file, and then then run this configuration from VSCode's Run and Debug tab. Note that while the `bundle_loader.js` is used to resolve imports, the debugger is executing the source under `dist/` - breakpoints must be placed in the appropriate files.

express-debug/bundle_loader.js

@@ -0,0 +1,60 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import * as path from "path";
+
+import yargs from "yargs";
+
+const argv = yargs(process.argv.slice(2)).option("bundle", {
+  describe: "JSON bundle describing application to host",
+  type: "string",
+  demandOption: true,
+}).argv;
+
+const bundlePath = argv.bundle;
+const bundle = JSON.parse(readFileSync(bundlePath, "utf-8"));
+
+const modules = {};
+for (let moduleDef of bundle.modules) {
+  modules[moduleDef.name] = moduleDef.module;
+}
+
+export async function resolve(specifier, context, nextResolve) {
+  const targetRoot = path.join(process.cwd(), `dist`);
+  let originalSpecifier = specifier;
+
+  if (context.parentURL) {
+    let canonicalSpecifier = context.parentURL;
+    if (context.parentURL.startsWith("file://")) {
+      canonicalSpecifier = context.parentURL.substr(7);
+    }
+    canonicalSpecifier = path.resolve(
+      path.dirname(canonicalSpecifier),
+      specifier
+    );
+    canonicalSpecifier = path.relative(targetRoot, canonicalSpecifier);
+    specifier = canonicalSpecifier;
+
+    // Result should not be relative
+    if (specifier.startsWith("../")) {
+      specifier = specifier.substr(3);
+    }
+  }
+
+  if (specifier in modules) {
+    const modulePath = path.join(targetRoot, specifier);
+    const targetDir = path.dirname(modulePath);
+    if (!existsSync(targetDir)) {
+      //console.debug(`Creating dir ${targetDir}`);
+      mkdirSync(targetDir, { recursive: true });
+    }
+
+    //console.debug(`Writing local module ${modulePath}`);
+    writeFileSync(modulePath, modules[specifier]);
+    return {
+      shortCircuit: true,
+      url: new URL(`file:${modulePath}`).href,
+      format: "module",
+    };
+  }
+
+  return nextResolve(originalSpecifier, context);
+}

express-debug/main.js

@@ -0,0 +1,190 @@
+// Take a bundle.json file, start a corresponding Express server
+
+// Use ccf polyfill to populate global ccf object
+import "@microsoft/ccf-app/polyfill.js";
+
+import { readFileSync, writeFileSync } from "fs";
+
+import express from "express";
+import yargs from "yargs";
+import pem from "pem";
+import https from "https";
+import jwt from "jsonwebtoken";
+
+const argv = yargs(process.argv.slice(2))
+  .option("bundle", {
+    describe: "JSON bundle describing application to host",
+    type: "string",
+    demandOption: true,
+  })
+  .option("cert", {
+    describe:
+      "Path where freshly created, self-signed service cert will be written",
+    type: "string",
+    default: "./service_cert.pem",
+  })
+  .option("port", {
+    describe: "Port to host server on",
+    type: "int",
+    default: 8000,
+  })
+  .strict().argv;
+
+const bundlePath = argv.bundle;
+const bundle = JSON.parse(readFileSync(bundlePath, "utf-8"));
+const metadata = bundle.metadata;
+
+let appRouter = express.Router();
+
+const routeConverter = {
+  // Convert an express-style templated path to a CCF path.
+  // eg /hello/:id/:place -> /hello/{id}/{place}
+  e2c: function (path) {
+    const re = /:([^\/]*)/g;
+    return path.replaceAll(re, "{$1}");
+  },
+
+  // Convert an CCF-style templated path to an express path.
+  // eg /hello/{id}/{place} -> /hello/:id/:place
+  c2e: function (path) {
+    const re = /{([^/]*)}/g;
+    return path.replaceAll(re, ":$1");
+  },
+};
+
+function ccfIdFromPem(cert) {
+  return cert.fingerprint256.replaceAll(":", "").toLowerCase();
+}
+
+function populateCaller(req, policies) {
+  // Note this does not _apply_ the auth policy. It only tries to construct a
+  // caller object of the correct shape, based on the specified policies and
+  // incoming request
+  for (let policy of policies) {
+    let caller = { policy: policy };
+    if (policy === "no_auth") {
+      return caller;
+    } else if (policy === "jwt") {
+      const authHeader = req.headers["authorization"];
+      if (authHeader && authHeader.startsWith("Bearer ")) {
+        const decoded = jwt.decode(authHeader.replace("Bearer ", ""), {
+          json: true,
+          complete: true,
+        });
+        caller.jwt = {
+          keyIssuer: "NOT VALIDATED",
+          header: decoded.header,
+          payload: decoded.payload,
+        };
+        return caller;
+      }
+    } else if (policy === "user_cert" || policy === "member_cert") {
+      const peerCert = req.client.getPeerX509Certificate();
+      if (peerCert !== undefined) {
+        caller.cert = peerCert;
+        caller.id = ccfIdFromPem(peerCert);
+        return caller;
+      }
+    } else if (policy === "user_cose_sign1") {
+      console.error("Igoring unimplemented user_cose_sign1 caller");
+      continue;
+    }
+  }
+
+  return null;
+}
+
+function ccfMiddleware(req, res, next) {
+  const rawBody = req.body;
+  req.body = {
+    text: function () {
+      return rawBody.toString("utf8");
+    },
+    json: function () {
+      return JSON.parse(rawBody);
+    },
+    arrayBuffer: function () {
+      return new ArrayBuffer(rawBody);
+    },
+  };
+
+  // CCF expects bare query string
+  req.query = req._parsedUrl.query || "";
+
+  next();
+}
+appRouter.use(
+  express.raw({
+    type: function (req) {
+      return true;
+    },
+  }),
+  ccfMiddleware
+);
+
+for (let [path, pathObject] of Object.entries(metadata.endpoints)) {
+  let route = appRouter.route(routeConverter.c2e(path));
+  for (let [method, methodObject] of Object.entries(pathObject)) {
+    route[method](async (req, res) => {
+      const module = await import(methodObject.js_module);
+      const handler = module[methodObject.js_function];
+
+      try {
+        // Convert route to CCF format
+        req.route = routeConverter.e2c(req.route.path);
+
+        // Populate req.caller
+        req.caller = populateCaller(req, methodObject.authn_policies);
+
+        // Call handler
+        const response = handler(req);
+
+        // TODO: Fill fields in same way CCF does
+        const statusCode = response.statusCode || (response.body ? 200 : 204);
+        res.status(statusCode);
+        res.set("content-type", "application/json");
+        res.send(response.body);
+      } catch (error) {
+        // TODO: Convert to CCF-style errors
+        console.error(`Error while executing ${method} ${path}: ${error}`);
+        res.status(500);
+        res.send(`Execution error: ${error}`);
+      }
+    });
+  }
+}
+
+// All CCF app endpoints are nested under /app - so tell express to
+// use our generated router at prefix "/app"
+const expressApp = express();
+expressApp.use("/app", appRouter);
+
+const port = argv.port;
+
+pem.createCertificate({ days: 1, selfSigned: true }, function (err, keys) {
+  if (err) {
+    console.error(`Error creating certificate: ${err.message}`);
+    process.exit(1);
+  }
+
+  console.log(`Writing server certificate to ${argv.cert}`);
+  writeFileSync(argv.cert, keys.certificate);
+
+  const server = https
+    .createServer(
+      {
+        key: keys.clientKey,
+        cert: keys.certificate,
+        // Store client cert if provided, but don't auto-reject it
+        requestCert: true,
+        rejectUnauthorized: false,
+      },
+      expressApp
+    )
+    .listen(port);
+  console.log(
+    `CCF express app listening on ${server.address().address}:${
+      server.address().port
+    }!`
+  );
+});

express-debug/package.json

@@ -0,0 +1,19 @@
+{
+  "private": true,
+  "name": "ccf-express-dbg",
+  "main": "main.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "engines": {
+    "node": ">=14"
+  },
+  "type": "module",
+  "dependencies": {
+    "@microsoft/ccf-app": "^3.0.12",
+    "express": "^4.18.2",
+    "jsonwebtoken": "^9.0.0",
+    "pem": "^1.14.8",
+    "yargs": "^17.7.2"
+  }
+}