From c6731e060986d2548ae5fd9df91ebe198cfa3a74 Mon Sep 17 00:00:00 2001 From: Eddy Ashton Date: Mon, 10 Jul 2023 16:09:29 +0000 Subject: [PATCH 1/4] Initial commit --- express-debug/README.md | 107 +++++++++++++++++++ express-debug/bundle_loader.js | 60 +++++++++++ express-debug/main.js | 190 +++++++++++++++++++++++++++++++++ express-debug/package.json | 19 ++++ 4 files changed, 376 insertions(+) create mode 100644 express-debug/README.md create mode 100644 express-debug/bundle_loader.js create mode 100644 express-debug/main.js create mode 100644 express-debug/package.json diff --git a/express-debug/README.md b/express-debug/README.md new file mode 100644 index 00000000..752f7ecd --- /dev/null +++ b/express-debug/README.md @@ -0,0 +1,107 @@ +# CCF Express + +## Summary +This folder contains a small tool to host a CCF JS app using ExpressJS. This uses the polyfills distributed with the `ccf-app` npm package to implement the `ccf` global, and exposes HTTP routes based on the metadata from an `app.json` file. + +The intention is that this can be used to debug CCF application handlers written in JavaScript or TypeScript. This does not start a multi-node network, or any enclaved code. Instead it starts a single local web server, attempting to provide the same `/app` HTTP API defined by `app.json`. + +### *WARNINGS* +This is a minimal PoC, and does not provide a perfectly matching server implementation. Notable discrepancies from a CCF node at the time of writing: +- No implementation of governance endpoints. This purely serves `/app` endpoints, and any attempts at governance will return 404s. Any app state which is bootstrapped by governance may need additional genesis calls. +- Does no authentication. It attempts to construct a valid `caller` object based on fields from the incoming request, but does not validate that this represents an identity that is known and trusted in the KV. +- KV does not offer opacity or snapshot isolation. Any writes to the KV are immediately visible to other concurrently executing handlers, and writes are _not_ rolled back for failing operations. +- Error response body shape is inconsistent. This does not produce JSON OData responses for errors in the same way a CCF node does. + +In short, this should only be used for testing happy-path execution flows. Any flows which rely on error responses or framework-inserted details are not supported out-of-the-box, and will require extensions to this tool. + +## Issues + +The approach used by this tool is to hook into the module loader (`--experimental-loader ./bundle_loader.js`) to resolve all imports directly from a single bundle file. This mirrors the CCF deployment path, ensuring all dependencies are atomically packaged, but is unnecessarily cumbersome, and leads to constant churn in the working `dist/` folder. Additionally it means the debugger is stepping into temporary files created within the `dist/` folder, rather than the original source files. + +It should instead be possible to import modules from their original locations on the local hard drive with standard import tooling, either with dynamic `import()` (as this does), or by generating the `main.js` application ahead-of-time. + +## Use + +The input file for this tool is a `bundle.json`, containing the routes exposed by a CCF app and all associated JS code. This object is the one emitted by the sample `build_bundle.js`, or contained in the `bundle` field of a `set_js_app` proposal. + +A minimal example looks something like this: +```json +{ + "metadata": { + "endpoints": { + "/increment/{id}": { + "post": { + "js_module": "endpoints/increment.js", + "js_function": "increment", + "forwarding_required": "sometimes", + "authn_policies": [], + "mode": "readwrite", + "openapi": {} + } + }, + "/count/{id}": { + "get": { + "js_module": "endpoints/count.js", + "js_function": "count", + "forwarding_required": "sometimes", + "authn_policies": [], + "mode": "readwrite", + "openapi": {} + } + } + } + }, + "modules": [ + { + "name": "endpoints/all.js", + "module": "export { increment } from './increment.js';\nexport { count } from './count.js';\n" + }, + { + "name": "endpoints/count.js", + "module": "import { ccf } from '../js/ccf-app/global.js';\n\nfunction count(request) {\n let counts = ccf.kv[\"counts\"];\n const id = ccf.jsonCompatibleToBuf(request.params.id);\n if (counts.has(id)) {\n return {\n body: {\n count: ccf.bufToJsonCompatible(counts.get(id))\n }\n };\n }\n else {\n return {\n statusCode: 404,\n body: `No count found for ${request.params.id}`\n };\n }\n}\n\nexport { count };\n" + }, + { + "name": "endpoints/increment.js", + "module": "import { ccf } from '../js/ccf-app/global.js';\n\nfunction increment(request) {\n let counts = ccf.kv[\"counts\"];\n const id = ccf.jsonCompatibleToBuf(request.params.id);\n const prevCount = counts.has(id) ? ccf.bufToJsonCompatible(counts.get(id)) : 0;\n counts.set(id, ccf.jsonCompatibleToBuf(prevCount + 1));\n return {};\n}\n\nexport { increment };\n" + }, + { + "name": "js/ccf-app/global.js", + "module": "// Copyright (c) Microsoft Corporation. All rights reserved.\n// Licensed under the Apache 2.0 License.\n/**\n * This module describes the global {@linkcode ccf} variable.\n * Direct access of this module or the {@linkcode ccf} variable is\n * typically not needed as all of its functionality is exposed\n * via other, often more high-level, modules.\n *\n * Accessing the {@linkcode ccf} global in a type-safe way is done\n * as follows:\n *\n * ```\n * import { ccf } from '@microsoft/ccf-app/global.js';\n * ```\n *\n * @module\n */\n// The global ccf variable and associated types are exported\n// as a regular module instead of using an ambient namespace\n// in a .d.ts definition file.\n// This avoids polluting the global namespace.\nconst ccf = globalThis.ccf;\n\nexport { ccf };\n" + } + ] +} +``` + +This contains metadata describing 2 endpoints (`POST /app/increment/{id}` and `GET /app/count/{id}`), the source files which implement them, and the modules they depend on. + +If the file above is in `my_bundle.json`, and we've run `npm install`, then this can be launched with: + +```bash +$ node --experimental-loader ./bundle_loader.js ./host.js --bundle ./my_bundle.json +Writing server certificate to ./service_cert.pem +CCF express app listening on :::8000! +``` + +Note this emits a new self-signed `service_cert.pem` used as the server identity. Clients must either obtain this, or disable TLS server authentication (`-k` to curl). + +Interact with the server like any other CCF node or web server: + +```bash +$ curl https://localhost:8000/app/count/foo --cacert ./service_cert.pem +No count found for foo + +$ curl https://localhost:8000/app/increment/foo -k -X POST +$ curl https://localhost:8000/app/count/foo -k +{"count":1} + +$ curl https://localhost:8000/app/increment/foo --cacert ./service_cert.pem -X POST +$ curl https://localhost:8000/app/increment/foo --cacert ./service_cert.pem -X POST +$ curl https://localhost:8000/app/count/foo --cacert ./service_cert.pem +{"count":3} +``` + +## VSCode integration + +This folder contains a [`.vscode/launch.json`](.vscode/launch.json) demonstrating how this tool may be hooked up to the VSCode debugger. + +To use this configuration, first select a `bundle.json` as the active file, and then then run this configuration from VSCode's Run and Debug tab. For instance if `my_bundle.json` is the active window, then this configuration will launch `node --experimental-loader ./bundle_loader.js ./host.js --bundle ./my_bundle.json` with the VSCode debugger attached. Note that while the `bundle_loader.js` is used to resolve imports, the debugger is executing the source under `dist/` - breakpoints must be placed in the appropriate file. \ No newline at end of file diff --git a/express-debug/bundle_loader.js b/express-debug/bundle_loader.js new file mode 100644 index 00000000..3180b801 --- /dev/null +++ b/express-debug/bundle_loader.js @@ -0,0 +1,60 @@ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; +import * as path from "path"; + +import yargs from "yargs"; + +const argv = yargs(process.argv.slice(2)).option("bundle", { + describe: "JSON bundle describing application to host", + type: "string", + demandOption: true, +}).argv; + +const bundlePath = argv.bundle; +const bundle = JSON.parse(readFileSync(bundlePath, "utf-8")); + +const modules = {}; +for (let moduleDef of bundle.modules) { + modules[moduleDef.name] = moduleDef.module; +} + +export async function resolve(specifier, context, nextResolve) { + const targetRoot = `${process.cwd()}/dist/`; + let originalSpecifier = specifier; + + if (context.parentURL) { + let canonicalSpecifier = context.parentURL; + if (context.parentURL.startsWith("file://")) { + canonicalSpecifier = context.parentURL.substr(7); + } + canonicalSpecifier = path.resolve( + path.dirname(canonicalSpecifier), + specifier + ); + canonicalSpecifier = path.relative(targetRoot, canonicalSpecifier); + specifier = canonicalSpecifier; + + // Result should not be relative + if (specifier.startsWith("../")) { + specifier = specifier.substr(3); + } + } + + if (specifier in modules) { + const modulePath = path.join(targetRoot, specifier); + const targetDir = path.dirname(modulePath); + if (!existsSync(targetDir)) { + //console.debug(`Creating dir ${targetDir}`); + mkdirSync(targetDir, { recursive: true }); + } + + //console.debug(`Writing local module ${modulePath}`); + writeFileSync(modulePath, modules[specifier]); + return { + shortCircuit: true, + url: new URL(`file:${modulePath}`).href, + format: "module", + }; + } + + return nextResolve(originalSpecifier, context); +} diff --git a/express-debug/main.js b/express-debug/main.js new file mode 100644 index 00000000..522a0ee3 --- /dev/null +++ b/express-debug/main.js @@ -0,0 +1,190 @@ +// Take a bundle.json file, start a corresponding Express server + +// Use ccf polyfill to populate global ccf object +import "@microsoft/ccf-app/polyfill.js"; + +import { readFileSync, writeFileSync } from "fs"; + +import express from "express"; +import yargs from "yargs"; +import pem from "pem"; +import https from "https"; +import jwt from "jsonwebtoken"; + +const argv = yargs(process.argv.slice(2)) + .option("bundle", { + describe: "JSON bundle describing application to host", + type: "string", + demandOption: true, + }) + .option("cert", { + describe: + "Path where freshly created, self-signed service cert will be written", + type: "string", + default: "./service_cert.pem", + }) + .option("port", { + describe: "Port to host server on", + type: "int", + default: 8000, + }) + .strict().argv; + +const bundlePath = argv.bundle; +const bundle = JSON.parse(readFileSync(bundlePath, "utf-8")); +const metadata = bundle.metadata; + +let appRouter = express.Router(); + +const routeConverter = { + // Convert an express-style templated path to a CCF path. + // eg /hello/:id/:place -> /hello/{id}/{place} + e2c: function (path) { + const re = /:([^\/]*)/g; + return path.replaceAll(re, "{$1}"); + }, + + // Convert an CCF-style templated path to an express path. + // eg /hello/{id}/{place} -> /hello/:id/:place + c2e: function (path) { + const re = /{([^/]*)}/g; + return path.replaceAll(re, ":$1"); + }, +}; + +function ccfIdFromPem(cert) { + return cert.fingerprint256.replaceAll(":", "").toLowerCase(); +} + +function populateCaller(req, policies) { + // Note this does not _apply_ the auth policy. It only tries to construct a + // caller object of the correct shape, based on the specified policies and + // incoming request + for (let policy of policies) { + let caller = { policy: policy }; + if (policy === "no_auth") { + return caller; + } else if (policy === "jwt") { + const authHeader = req.headers["authorization"]; + if (authHeader && authHeader.startsWith("Bearer ")) { + const decoded = jwt.decode(authHeader.replace("Bearer ", ""), { + json: true, + complete: true, + }); + caller.jwt = { + keyIssuer: "NOT VALIDATED", + header: decoded.header, + payload: decoded.payload, + }; + return caller; + } + } else if (policy === "user_cert" || policy === "member_cert") { + const peerCert = req.client.getPeerX509Certificate(); + if (peerCert !== undefined) { + caller.cert = peerCert; + caller.id = ccfIdFromPem(peerCert); + return caller; + } + } else if (policy === "user_cose_sign1") { + console.error("Igoring unimplemented user_cose_sign1 caller"); + continue; + } + } + + return null; +} + +function ccfMiddleware(req, res, next) { + const rawBody = req.body; + req.body = { + text: function () { + return rawBody.toString("utf8"); + }, + json: function () { + return JSON.parse(rawBody); + }, + arrayBuffer: function () { + return new ArrayBuffer(rawBody); + }, + }; + + // CCF expects bare query string + req.query = req._parsedUrl.query || ""; + + next(); +} +appRouter.use( + express.raw({ + type: function (req) { + return true; + }, + }), + ccfMiddleware +); + +for (let [path, pathObject] of Object.entries(metadata.endpoints)) { + let route = appRouter.route(routeConverter.c2e(path)); + for (let [method, methodObject] of Object.entries(pathObject)) { + route[method](async (req, res) => { + const module = await import(methodObject.js_module); + const handler = module[methodObject.js_function]; + + try { + // Convert route to CCF format + req.route = routeConverter.e2c(req.route.path); + + // Populate req.caller + req.caller = populateCaller(req, methodObject.authn_policies); + + // Call handler + const response = handler(req); + + // TODO: Fill fields in same way CCF does + const statusCode = response.statusCode || (response.body ? 200 : 204); + res.status(statusCode); + res.set("content-type", "application/json"); + res.send(response.body); + } catch (error) { + // TODO: Convert to CCF-style errors + console.error(`Error while executing ${method} ${path}: ${error}`); + res.status(500); + res.send(`Execution error: ${error}`); + } + }); + } +} + +// All CCF app endpoints are nested under /app - so tell express to +// use our generated router at prefix "/app" +const expressApp = express(); +expressApp.use("/app", appRouter); + +const port = argv.port; + +pem.createCertificate({ days: 1, selfSigned: true }, function (err, keys) { + if (err) { + console.error(`Error creating certificate: ${err.message}`); + process.exit(1); + } + + console.log(`Writing server certificate to ${argv.cert}`); + writeFileSync(argv.cert, keys.certificate); + + const server = https + .createServer( + { + key: keys.clientKey, + cert: keys.certificate, + // Store client cert if provided, but don't auto-reject it + requestCert: true, + rejectUnauthorized: false, + }, + expressApp + ) + .listen(port); + console.log( + `CCF express app listening on ${server.address().address}:${ + server.address().port + }!` + ); +}); diff --git a/express-debug/package.json b/express-debug/package.json new file mode 100644 index 00000000..97862b0d --- /dev/null +++ b/express-debug/package.json @@ -0,0 +1,19 @@ +{ + "private": true, + "name": "ccf-express-dbg", + "main": "main.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "engines": { + "node": ">=14" + }, + "type": "module", + "dependencies": { + "@microsoft/ccf-app": "^3.0.12", + "express": "^4.18.2", + "jsonwebtoken": "^9.0.0", + "pem": "^1.14.8", + "yargs": "^17.7.2" + } +} \ No newline at end of file From ac51544734038bcbcce79e5c2aa0fb4e5e6adb79 Mon Sep 17 00:00:00 2001 From: Eddy Ashton Date: Mon, 10 Jul 2023 16:09:44 +0000 Subject: [PATCH 2/4] Intellisense for rollup config --- banking-app/rollup.config.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/banking-app/rollup.config.js b/banking-app/rollup.config.js index 332d8df0..f05ef544 100644 --- a/banking-app/rollup.config.js +++ b/banking-app/rollup.config.js @@ -2,6 +2,9 @@ import { nodeResolve } from "@rollup/plugin-node-resolve"; import commonjs from "@rollup/plugin-commonjs"; import typescript from "@rollup/plugin-typescript"; +/** + * @type {import('rollup').RollupOptions} + */ export default { input: "src/endpoints/all.ts", output: { From 0449dea30c8b01ba5e74ea30e925acc73d8fb11b Mon Sep 17 00:00:00 2001 From: Eddy Ashton Date: Mon, 10 Jul 2023 16:48:33 +0000 Subject: [PATCH 3/4] Mildly less POSIX --- express-debug/bundle_loader.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/express-debug/bundle_loader.js b/express-debug/bundle_loader.js index 3180b801..2dbcdf3a 100644 --- a/express-debug/bundle_loader.js +++ b/express-debug/bundle_loader.js @@ -18,7 +18,7 @@ for (let moduleDef of bundle.modules) { } export async function resolve(specifier, context, nextResolve) { - const targetRoot = `${process.cwd()}/dist/`; + const targetRoot = path.join(process.cwd(), `dist`); let originalSpecifier = specifier; if (context.parentURL) { From 6d87715e03ba3d964b4a48a089af6a57d414187d Mon Sep 17 00:00:00 2001 From: Eddy Ashton Date: Mon, 10 Jul 2023 16:55:30 +0000 Subject: [PATCH 4/4] Update README --- express-debug/README.md | 96 +++++++++++++---------------------------- 1 file changed, 31 insertions(+), 65 deletions(-) diff --git a/express-debug/README.md b/express-debug/README.md index 752f7ecd..3915ea3a 100644 --- a/express-debug/README.md +++ b/express-debug/README.md @@ -16,7 +16,7 @@ In short, this should only be used for testing happy-path execution flows. Any f ## Issues -The approach used by this tool is to hook into the module loader (`--experimental-loader ./bundle_loader.js`) to resolve all imports directly from a single bundle file. This mirrors the CCF deployment path, ensuring all dependencies are atomically packaged, but is unnecessarily cumbersome, and leads to constant churn in the working `dist/` folder. Additionally it means the debugger is stepping into temporary files created within the `dist/` folder, rather than the original source files. +The approach used by this tool is to hook into the module loader (`--experimental-loader ./bundle_loader.js`) to resolve all imports directly from a single bundle file. This mirrors the CCF deployment path, ensuring all dependencies are atomically packaged, but is unnecessarily cumbersome, and leads to constant rewriting of files into a temporary `dist/` folder. Additionally it means the debugger is stepping into temporary files created within the `dist/` folder, rather than the original source files. It should instead be possible to import modules from their original locations on the local hard drive with standard import tooling, either with dynamic `import()` (as this does), or by generating the `main.js` application ahead-of-time. @@ -24,60 +24,16 @@ It should instead be possible to import modules from their original locations on The input file for this tool is a `bundle.json`, containing the routes exposed by a CCF app and all associated JS code. This object is the one emitted by the sample `build_bundle.js`, or contained in the `bundle` field of a `set_js_app` proposal. -A minimal example looks something like this: -```json -{ - "metadata": { - "endpoints": { - "/increment/{id}": { - "post": { - "js_module": "endpoints/increment.js", - "js_function": "increment", - "forwarding_required": "sometimes", - "authn_policies": [], - "mode": "readwrite", - "openapi": {} - } - }, - "/count/{id}": { - "get": { - "js_module": "endpoints/count.js", - "js_function": "count", - "forwarding_required": "sometimes", - "authn_policies": [], - "mode": "readwrite", - "openapi": {} - } - } - } - }, - "modules": [ - { - "name": "endpoints/all.js", - "module": "export { increment } from './increment.js';\nexport { count } from './count.js';\n" - }, - { - "name": "endpoints/count.js", - "module": "import { ccf } from '../js/ccf-app/global.js';\n\nfunction count(request) {\n let counts = ccf.kv[\"counts\"];\n const id = ccf.jsonCompatibleToBuf(request.params.id);\n if (counts.has(id)) {\n return {\n body: {\n count: ccf.bufToJsonCompatible(counts.get(id))\n }\n };\n }\n else {\n return {\n statusCode: 404,\n body: `No count found for ${request.params.id}`\n };\n }\n}\n\nexport { count };\n" - }, - { - "name": "endpoints/increment.js", - "module": "import { ccf } from '../js/ccf-app/global.js';\n\nfunction increment(request) {\n let counts = ccf.kv[\"counts\"];\n const id = ccf.jsonCompatibleToBuf(request.params.id);\n const prevCount = counts.has(id) ? ccf.bufToJsonCompatible(counts.get(id)) : 0;\n counts.set(id, ccf.jsonCompatibleToBuf(prevCount + 1));\n return {};\n}\n\nexport { increment };\n" - }, - { - "name": "js/ccf-app/global.js", - "module": "// Copyright (c) Microsoft Corporation. All rights reserved.\n// Licensed under the Apache 2.0 License.\n/**\n * This module describes the global {@linkcode ccf} variable.\n * Direct access of this module or the {@linkcode ccf} variable is\n * typically not needed as all of its functionality is exposed\n * via other, often more high-level, modules.\n *\n * Accessing the {@linkcode ccf} global in a type-safe way is done\n * as follows:\n *\n * ```\n * import { ccf } from '@microsoft/ccf-app/global.js';\n * ```\n *\n * @module\n */\n// The global ccf variable and associated types are exported\n// as a regular module instead of using an ambient namespace\n// in a .d.ts definition file.\n// This avoids polluting the global namespace.\nconst ccf = globalThis.ccf;\n\nexport { ccf };\n" - } - ] -} -``` +For instance, to run the `banking-app` sample contained in this repo: + +```bash +$ cd banking-app -This contains metadata describing 2 endpoints (`POST /app/increment/{id}` and `GET /app/count/{id}`), the source files which implement them, and the modules they depend on. +$ npm run build -If the file above is in `my_bundle.json`, and we've run `npm install`, then this can be launched with: +$ cd ../express-debug -```bash -$ node --experimental-loader ./bundle_loader.js ./host.js --bundle ./my_bundle.json +$ node --experimental-loader ./bundle_loader.js ./main.js --bundle ../banking-app/dist/bundle.json Writing server certificate to ./service_cert.pem CCF express app listening on :::8000! ``` @@ -87,21 +43,31 @@ Note this emits a new self-signed `service_cert.pem` used as the server identity Interact with the server like any other CCF node or web server: ```bash -$ curl https://localhost:8000/app/count/foo --cacert ./service_cert.pem -No count found for foo - -$ curl https://localhost:8000/app/increment/foo -k -X POST -$ curl https://localhost:8000/app/count/foo -k -{"count":1} - -$ curl https://localhost:8000/app/increment/foo --cacert ./service_cert.pem -X POST -$ curl https://localhost:8000/app/increment/foo --cacert ./service_cert.pem -X POST -$ curl https://localhost:8000/app/count/foo --cacert ./service_cert.pem -{"count":3} +$ curl https://localhost:8000/app/balance/bob --cacert ./service_cert.pem ``` ## VSCode integration -This folder contains a [`.vscode/launch.json`](.vscode/launch.json) demonstrating how this tool may be hooked up to the VSCode debugger. +To launch this tool with the VSCode debugger attached, add a VSCode launch configuration (to `.vscode/launch.json`) based on the following snippet: + +```json +{ + "configurations": [ + { + "type": "node", + "request": "launch", + "cwd": "${workspaceFolder}/express-debug/", + "name": "Host current bundle with ExpressJS", + "skipFiles": ["/**"], + "args": ["--bundle", "${file}"], + "runtimeArgs": [ + "--experimental-loader", + "${workspaceFolder}/express-debug/bundle_loader.js" + ], + "program": "${workspaceFolder}/express-debug/main.js" + } + ] +} +``` -To use this configuration, first select a `bundle.json` as the active file, and then then run this configuration from VSCode's Run and Debug tab. For instance if `my_bundle.json` is the active window, then this configuration will launch `node --experimental-loader ./bundle_loader.js ./host.js --bundle ./my_bundle.json` with the VSCode debugger attached. Note that while the `bundle_loader.js` is used to resolve imports, the debugger is executing the source under `dist/` - breakpoints must be placed in the appropriate file. \ No newline at end of file +To use this configuration, first select a `bundle.json` as the active file, and then then run this configuration from VSCode's Run and Debug tab. Note that while the `bundle_loader.js` is used to resolve imports, the debugger is executing the source under `dist/` - breakpoints must be placed in the appropriate files. \ No newline at end of file