feature: create inflightRequestThrottle plugin (restify#1431)

docs/api/plugins.md

@@ -153,6 +153,12 @@ event, e.g., `server.on('after', plugins.metrics());`:
   * `res` {Object} the response obj
   * `route` {Object} the route obj that serviced the request
 
+The module includes the following plugins to be used with restify's `pre` event:
+* `inflightRequestThrottle(options)` - limits the max number of inflight requests
+  * `options.limit` {Number} the maximum number of inflight requests the server will handle before returning an error
+  * `options.err` {Error} opts.err A restify error used as a response when the inflight request limit is exceeded
+  * `options.server` {Object} The restify server that this module will throttle
+
 ## Accept Parser
 
 Parses out the `Accept` header, and ensures that the server can respond to what
@@ -439,6 +445,31 @@ uniform request distribution.  To enable this, you can pass in
 `options.tokensTable`, which is simply any Object that supports `put` and `get`
 with a `String` key, and an `Object` value.
 
+## Inflight Request Throttling
+
+```js
+var errors = require('restify-errors');
+var restify = require('restify');
+
+var server = restify.createServer();
+const options = { limit: 600, server: server };
+options.res = new errors.InternalServerError();
+server.pre(restify.plugins.inflightRequestThrottle(options));
+```
+
+The `inflightRequestThrottle` module allows you to specify an upper limit to
+the maximum number of inflight requests your server is able to handle. This
+is a simple heuristic for protecting against event loop contention between
+requests causing unacceptable latencies.
+
+The custom error is optional, and allows you to specify your own response
+and status code when rejecting incoming requests due to too many inflight
+requests. It defaults to `503 ServiceUnavailableError`.
+
+This plugin should be registered as early as possibly in the middleware stack
+using `pre` to avoid performing unnecessary work.
+
+
 ## Conditional Request Handler
 
 ```js

lib/plugins/index.js

@@ -14,6 +14,7 @@ module.exports = {
     dateParser: require('./date'),
     fullResponse: require('./fullResponse'),
     gzipResponse: require('./gzip'),
+    inflightRequestThrottle: require('./inflightRequestThrottle'),
     jsonBodyParser: require('./jsonBodyParser'),
     jsonp: require('./jsonp'),
     multipartBodyParser: require('./multipartBodyParser'),

lib/plugins/inflightRequestThrottle.js

@@ -0,0 +1,64 @@
+'use strict';
+
+var assert = require('assert-plus');
+var ServiceUnavailableError = require('restify-errors').ServiceUnavailableError;
+var defaultResponse = new ServiceUnavailableError('resource exhausted');
+
+/**
+ * inflightRequestThrottle
+ *
+ * Place an upper limit on the number of inlfight requests restify will accept.
+ * For every request that exceeds this threshold, restify will respond with an
+ * error. This plugin should be registered as early as possible in the
+ * middleware stack using `pre` to avoid performing unnecessary work.
+ *
+ * @param {Object} opts configure this plugin
+ * @param {Number} opts.limit maximum number of inflight requests the server
+ *    will handle before returning an error
+ * @param {Error} opts.err A restify error used as a response when the inflight
+ *    request limit is exceeded
+ * @param {Function} opts.server the instance of the restify server this plugin
+ *    will throttle.
+ * @returns {Function} middleware to be registered on server.pre
+ */
+function inflightRequestThrottle (opts) {
+
+    // Scrub input and populate our configuration
+    assert.object(opts, 'opts');
+    assert.number(opts.limit, 'opts.limit');
+    assert.object(opts.server, 'opts.server');
+    assert.func(opts.server.inflightRequests, 'opts.server.inflightRequests');
+
+    if (opts.err !== undefined && opts.err !== null) {
+        assert.ok(opts.err instanceof Error, 'opts.res must be an error');
+        assert.optionalNumber(opts.err.statusCode, 'opts.err.statusCode');
+    }
+
+    var self = {};
+    self._err = opts.err || defaultResponse;
+    self._limit = opts.limit;
+    self._server = opts.server;
+
+    function onRequest (req, res, next) {
+        var inflightRequests = self._server.inflightRequests();
+
+        if (inflightRequests > self._limit) {
+            req.log.trace({
+                plugin: 'inflightRequestThrottle',
+                inflightRequests: inflightRequests,
+                limit: self._limit
+            }, 'maximum inflight requests exceeded, rejecting request');
+            return res.send(self._err);
+        }
+
+        return next();
+    }
+
+    // We need to bind in order to keep our `this` context when passed back
+    // out of the constructor.
+    return onRequest;
+}
+
+inflightRequestThrottle.prototype.onRequest =
+
+module.exports = inflightRequestThrottle;

test/plugins/inflightRequestThrottle.test.js

@@ -0,0 +1,123 @@
+'use strict';
+
+var assert = require('chai').assert;
+var restify = require('../../lib/index.js');
+var restifyClients = require('restify-clients');
+var inflightRequestThrottle = restify.plugins.inflightRequestThrottle;
+
+function fakeServer(count) {
+    return {
+        inflightRequests: function () {
+            return count;
+        }
+    };
+}
+
+describe('inlfightRequestThrottle', function () {
+
+    it('Unit: Should shed load', function (done) {
+        var logged = false;
+        var opts = { server: fakeServer(10), limit: 1 };
+        var plugin = inflightRequestThrottle(opts);
+        function send (body) {
+            assert(logged, 'Should have emitted a log');
+            assert.equal(body.statusCode, 503, 'Defaults to 503 status');
+            assert(body instanceof Error, 'Defaults to error body');
+            done();
+        }
+        function next () {
+            assert(false, 'Should not call next');
+            done();
+        }
+        function trace () {
+            logged = true;
+        }
+        var log = { trace: trace };
+        var fakeReq = { log: log };
+        plugin(fakeReq, { send: send }, next);
+    });
+
+    it('Unit: Should support custom response', function (done) {
+        var server = fakeServer(10);
+        var err = new Error('foo');
+        var opts = { server: server, limit: 1, err: err };
+        var plugin = inflightRequestThrottle(opts);
+        function send (body) {
+            assert.equal(body, err, 'Overrides body');
+            done();
+        }
+        function next () {
+            assert(false, 'Should not call next');
+            done();
+        }
+        var fakeReq = { log : { trace: function () {} } };
+        plugin(fakeReq, { send: send }, next);
+    });
+
+    it('Unit: Should let request through when not under load', function (done) {
+        var opts = { server: fakeServer(1), limit: 2 };
+        var plugin = inflightRequestThrottle(opts);
+        function send () {
+            assert(false, 'Should not call send');
+            done();
+        }
+        function next () {
+            assert(true, 'Should call next');
+            done();
+        }
+        var fakeReq = { log : { trace: function () {} } };
+        plugin(fakeReq, { send: send }, next);
+    });
+
+    it('Integration: Should shed load', function (done) {
+        var server = restify.createServer();
+        var client = {
+            close: function () {}
+        };
+        var isDone = false;
+        var to;
+        function finish() {
+            if (isDone) {
+                return null;
+            }
+            clearTimeout(to);
+            isDone = true;
+            client.close();
+            server.close();
+            return done();
+        }
+        to = setTimeout(finish, 2000);
+        var err = new Error('foo');
+        err.statusCode = 555;
+        var opts = { server: server, limit: 1, err: err };
+        server.pre(inflightRequestThrottle(opts));
+        var RES;
+        server.get('/foo', function (req, res) {
+            if (RES) {
+                res.send(999);
+            } else {
+                RES = res;
+            }
+        });
+        server.listen(0, '127.0.0.1', function () {
+            client = restifyClients.createJsonClient({
+                url: 'http://127.0.0.1:' + server.address().port,
+                retry: false
+            });
+            client.get({ path: '/foo' }, function (e, _, res) {
+                assert(e === null || e === undefined,
+                    'First request isnt shed');
+                assert.equal(res.statusCode, 200, '200 returned on success');
+                finish();
+            });
+            client.get({ path: '/foo' }, function (e, _, res) {
+                assert(e, 'Second request is shed');
+                assert.equal(e.name,
+                    'InternalServerError', 'Default err returned');
+                assert.equal(res.statusCode, 555,
+                    'Default shed status code returned');
+                RES.send(200);
+            });
+        });
+    });
+});