fix: gracefully handle out of band errors

package.json

@@ -61,10 +61,11 @@
     "fs-extra": "^5.0.0",
     "glob": "^7.1.2",
     "htmlparser2": "^3.9.2",
+    "http-cache-semantics": "^3.8.1",
     "import-cwd": "^2.1.0",
     "ioredis": "^3.2.2",
     "ioredis-mock": "^3.6.1",
-    "isolated-vm": "^1.0.0",
+    "isolated-vm": "github:laverdet/isolated-vm#master",
     "js-yaml": "^3.10.0",
     "memory-fs": "^0.4.1",
     "mksuid": "0.0.3",
@@ -73,6 +74,7 @@
     "promise.prototype.finally": "^3.1.0",
     "promptly": "^3.0.3",
     "readable-stream": "^2.3.3",
+    "spark-md5": "^3.0.0",
     "string-to-stream": "^1.1.0",
     "text-encoding": "^0.6.4",
     "to-arraybuffer": "^1.0.1",
@@ -82,8 +84,6 @@
     "web-streams-polyfill": "^1.3.2",
     "webpack": "^3.10.0",
     "whatwg-url": "^6.4.0",
-    "http-cache-semantics": "^3.8.1",
-    "spark-md5": "^3.0.0",
     "winston": "^2.4.0",
     "ws": "^4.0.0"
   },

src/bridge/bridge.ts

@@ -1,5 +1,6 @@
 import './proxy_stream'
 import './fetch'
+import './heap'
 import './formdata'
 import './fly/cache'
 import * as ivm from 'isolated-vm'

src/bridge/fetch.ts

@@ -15,8 +15,6 @@ import { Trace } from '../trace'
 const fetchAgent = new http.Agent({ keepAlive: true });
 const fetchHttpsAgent = new https.Agent({ keepAlive: true, rejectUnauthorized: false })
 
-const wormholeRegex = /^(wormhole\.)/
-
 registerBridge('fetch', fetchBridge)
 
 export function fetchBridge(ctx: Context) {
@@ -31,7 +29,7 @@ export function fetchBridge(ctx: Context) {
     log.debug("fetch depth: ", depth)
     if (depth >= 3) {
       log.error("too much recursion: ", depth)
-      ctx.applyCallback(cb, ["Too much recursion"])
+      ctx.tryCallback(cb, ["Too much recursion"])
       return
     }
 
@@ -109,19 +107,23 @@ export function fetchBridge(ctx: Context) {
               setImmediate(async () => {
                 res.on("close", function () {
                   t.end()
-                  callback.apply(undefined, ["close"])
+                  ctx.addCallback(callback)
+                  ctx.applyCallback(callback, ["close"])
                 })
                 res.on("end", function () {
                   t.end()
-                  callback.apply(undefined, ["end"])
+                  ctx.addCallback(callback)
+                  ctx.applyCallback(callback, ["end"])
                 })
                 res.on("error", function (err: Error) {
                   t.end()
-                  callback.apply(undefined, ["error", err.toString()])
+                  ctx.addCallback(callback)
+                  ctx.applyCallback(callback, ["error", err.toString()])
                 })
 
                 res.on("data", function (data: Buffer) {
-                  callback.apply(undefined, ["data", transferInto(data)])
+                  ctx.addCallback(callback)
+                  ctx.applyCallback(callback, ["data", transferInto(data)])
                 })
                 res.resume()
               })
@@ -137,7 +139,7 @@ export function fetchBridge(ctx: Context) {
 
       req.on("error", function (err) {
         log.error("error requesting http resource", err)
-        ctx.applyCallback(cb, [err.toString()])
+        ctx.tryCallback(cb, [err.toString()])
       })
 
       req.end(body && Buffer.from(body) || null)

src/bridge/fly/cache.ts

@@ -14,7 +14,7 @@ registerBridge('flyCacheSet', function (ctx: Context, config: Config) {
     let k = "cache:" + ctx.meta.get('app').id + ":" + key
 
     if (!config.cacheStore)
-      return ctx.applyCallback(callback, [errCacheStoreUndefined.toString()])
+      return ctx.tryCallback(callback, [errCacheStoreUndefined.toString()])
 
     let size = 0
     if (value instanceof ArrayBuffer) {
@@ -26,13 +26,13 @@ registerBridge('flyCacheSet', function (ctx: Context, config: Config) {
     }
     config.cacheStore.set(k, value, ttl).then((ok) => {
       t.end({ size: size, key: key })
-      ctx.applyCallback(callback, [null, ok])
+      ctx.tryCallback(callback, [null, ok])
     }).catch((err) => {
       log.error(err)
       t.end()
-      ctx.applyCallback(callback, [err.toString()])
+      ctx.tryCallback(callback, [err.toString()])
     })
-    return callback
+    return
   }
 })
 
@@ -43,14 +43,14 @@ registerBridge('flyCacheExpire', function (ctx: Context, config: Config) {
     let k = "cache:" + ctx.meta.get('app').id + ":" + key
 
     if (!config.cacheStore)
-      return ctx.applyCallback(callback, [errCacheStoreUndefined.toString()])
+      return ctx.tryCallback(callback, [errCacheStoreUndefined.toString()])
 
     config.cacheStore.expire(k, ttl).then((ok) => {
       t.end({ key: key })
       ctx.applyCallback(callback, [null, ok])
     }).catch((err) => {
       t.end()
-      ctx.applyCallback(callback, [err.toString()])
+      ctx.tryCallback(callback, [err.toString()])
     })
   }
 })
@@ -62,7 +62,7 @@ registerBridge('flyCacheGet', function (ctx: Context, config: Config) {
     let k = "cache:" + ctx.meta.get('app').id + ":" + key
 
     if (!config.cacheStore)
-      return ctx.applyCallback(callback, [errCacheStoreUndefined.toString()])
+      return ctx.tryCallback(callback, [errCacheStoreUndefined.toString()])
 
     config.cacheStore.get(k).then((buf) => {
       const size = buf ? buf.byteLength : 0
@@ -71,7 +71,7 @@ registerBridge('flyCacheGet', function (ctx: Context, config: Config) {
     }).catch((err) => {
       console.error("got err in cache.get", err)
       t.end()
-      ctx.applyCallback(callback, [err.toString()])
+      ctx.tryCallback(callback, [err.toString()])
     })
   }
 })

src/bridge/heap.ts

@@ -0,0 +1,15 @@
+import { registerBridge, Context } from './'
+
+import { ivm } from '../'
+import log from "../log"
+
+registerBridge('getHeapStatistics', function (ctx: Context) {
+  return function (callback: ivm.Reference<Function>) {
+    ctx.addCallback(callback)
+    ctx.iso.getHeapStatistics().then((heap) => {
+      ctx.tryCallback(callback, [null, new ivm.ExternalCopy(heap).copyInto()])
+    }).catch((err) => {
+      ctx.tryCallback(callback, [err.toString()])
+    })
+  }
+})

src/bridge/index.ts

@@ -9,8 +9,10 @@ export function registerBridge(name: string, fn: BridgeFunctionFactory) {
 export interface Context {
   meta: Map<string, any>
   trace: Trace | undefined
+  iso: ivm.Isolate,
   addCallback(fn: ivm.Reference<Function>): any
-  applyCallback(fn: ivm.Reference<Function>, args: any[]): Promise<any>
+  applyCallback(fn: ivm.Reference<Function>, args: any[]): Promise<void>
+  tryCallback(fn: ivm.Reference<Function>, args: any[]): Promise<void>
 }
 
 export interface BridgeFunctionFactory {

src/cmd/server.ts

@@ -27,6 +27,9 @@ root
     const { Server } = require('../server')
     const { DefaultContextStore } = require('../default_context_store');
 
+    process.on('uncaughtException', err => log.error('uncaught exception:', err, err.stack));
+    process.on('unhandledRejection', err => log.error('unhandled rejection:', err, err.stack));
+
     const cwd = process.cwd()
     let conf = parseConfig(cwd)
 

src/context.ts

@@ -16,6 +16,8 @@ export class Context extends EventEmitter {
 	intervals: { [id: number]: NodeJS.Timer }
 	callbacks: ivm.Reference<Function>[]
 
+	fireEventFn?: ivm.Reference<Function>
+
 	iso: ivm.Isolate
 
 	private currentTimerId: number;
@@ -35,18 +37,39 @@ export class Context extends EventEmitter {
 	addCallback(fn: ivm.Reference<Function>) {
 		this.callbacks.push(fn)
 		this.emit("callbackAdded", fn)
-		log.debug("CALLBACK ADDED")
+		log.debug("Added a callback")
 	}
 
-	async applyCallback(fn: ivm.Reference<Function>, args: any[]) {
+	async applyCallback(fn: ivm.Reference<Function>, args: any[], opts?: any) {
 		try {
-			await fn.apply(null, args)
+			return await this._applyCallback(fn, args, opts)
+		} catch (err) {
+			log.error("Caught error while calling back:", err)
+			this.emit("error", err)
+		}
+	}
+
+	async _applyCallback(fn: ivm.Reference<Function>, args: any[], opts?: any) {
+		log.debug("Applying callback to context.")
+		try {
+			if (this.iso.isDisposed)
+				return
+			return await fn.apply(null, args, opts)
 		} finally {
 			const i = this.callbacks.indexOf(fn)
 			if (i !== -1) {
 				this.callbacks.splice(i, 1)
 				this.emit("callbackApplied")
 			}
+			log.debug("Done with callback.")
+		}
+	}
+
+	async tryCallback(fn: ivm.Reference<Function>, args: any[], opts?: any) {
+		try {
+			return await this._applyCallback(fn, args, opts)
+		} catch (err) {
+			log.error("Error trying to apply callback", err)
 		}
 	}
 
@@ -93,11 +116,31 @@ export class Context extends EventEmitter {
 		}
 	}
 
+	async fireEvent(name: string, args: any[], opts?: any) {
+		if (this.iso.isDisposed)
+			throw new Error("Isolate is disposed or disposing.")
+		try {
+			if (!this.fireEventFn)
+				this.fireEventFn = await this.get("fireEvent")
+			log.debug("Firing event", name)
+			return await this.fireEventFn.apply(null, [name, ...args], opts)
+		} catch (err) {
+			log.error("Error firing event:", err)
+			this.emit("error", err)
+		}
+	}
+
 	async get(name: string) {
+		if (this.iso.isDisposed)
+			throw new Error("Isolate is disposed or disposing.")
+		log.debug("Getting global", name)
 		return await this.global.get(name)
 	}
 
 	async set(name: any, value: any) {
+		if (this.iso.isDisposed)
+			throw new Error("Isolate is disposed or disposing.")
+		log.debug("Setting global", name)
 		return await this.global.set(name, value)
 	}
 

src/default_context_store.ts

@@ -52,15 +52,15 @@ export class DefaultContextStore implements ContextStore {
   }
 
   async getIsolate() {
-    if (this.isolate)
+    if (this.isolate && !this.isolate.isDisposed)
       return this.isolate
     await v8Env.waitForReadiness()
     this.resetIsolate()
     return this.isolate
   }
 
   resetIsolate() {
-    if (this.isolate)
+    if (this.isolate && !this.isolate.isDisposed)
       this.isolate.dispose()
     this.isolate = new ivm.Isolate({
       snapshot: v8Env.snapshot,