From 374c1884e3beaa3306c572c2e48a5837167f5b9b Mon Sep 17 00:00:00 2001 From: Danny Guo Date: Tue, 21 Jul 2015 16:04:02 -0400 Subject: [PATCH 1/4] Add support for transparent key prefixing. Close #95 --- README.md | 15 +++++-- lib/command.js | 41 +++++++++++++++++++ lib/commander.js | 6 ++- lib/redis.js | 4 +- lib/script.js | 6 ++- test/functional/pipeline.js | 15 +++++++ test/functional/scripting.js | 14 +++++++ test/functional/send_command.js | 71 +++++++++++++++++++++++++++++++++ 8 files changed, 166 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 30841dc5..2a8a8c6d 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ A delightful, performance-focused Redis client for Node and io.js Support Redis >= 2.6.12 and (Node.js >= 0.10.16 or io.js). -# Feature +# Features ioredis is a robust, full-featured Redis client used in the world's biggest online commerce company [Alibaba](http://www.alibaba.com/). @@ -24,6 +24,7 @@ used in the world's biggest online commerce company [Alibaba](http://www.alibaba 0. Supports offline queue and ready checking. 0. Supports ES6 types such as `Map` and `Set`. 0. Sophisticated error handling strategy. +0. Transparent key prefixing.
@@ -625,6 +626,16 @@ If [hiredis](https://github.com/redis/hiredis-node) is installed(by `npm install ioredis will use it by default. Otherwise, a pure JavaScript parser will be used. Typically there's not much differences between them in terms of performance. +## Transparent Key Prefixing +This feature allows you to specify a string that will automatically be prepended +to all the keys in a command, which makes it easier to manage your key +namespaces. + +```javascript +var fooRedis = new Redis({ keyPrefix: 'foo:' }); +fooRedis.set('bar', 'baz'); // Actually sends SET foo:bar baz +``` +
# Error Handling @@ -808,8 +819,6 @@ Ordered by date of first contribution. [Auto-generated](https://github.com/dtrej # Roadmap -* Transparent Key Prefixing - # Acknowledge The JavaScript and hiredis parsers are modified from [node_redis](https://github.com/mranney/node_redis) (MIT License, Copyright (c) 2010 Matthew Ranney, http://ranney.com/). diff --git a/lib/command.js b/lib/command.js index a730311d..5885cf04 100644 --- a/lib/command.js +++ b/lib/command.js @@ -43,6 +43,11 @@ function Command(name, args, options, callback) { this.replyEncoding = options && options.replyEncoding; this.errorStack = options && options.errorStack; this.args = args ? _.flatten(args) : []; + if (options && options.keyPrefix) { + this._iterateKeys(function (key) { + return options.keyPrefix + key; + }); + } this.callback = callback; this.initPromise(); } @@ -83,6 +88,18 @@ Command.prototype.getSlot = function () { }; Command.prototype.getKeys = function () { + return this._iterateKeys(); +}; + +/** + * Iterate through the command arguments that are considered keys. + * + * @param {function} [transform] - The transformation that should be applied to + * each key. The transformations will persist. + * @return {string[]} The keys of the command. + * @private + */ +Command.prototype._iterateKeys = function (transform) { if (typeof this._keys === 'undefined') { this._keys = []; var i, keyStart, keyStop; @@ -93,10 +110,16 @@ Command.prototype.getKeys = function () { case 'evalsha': keyStop = parseInt(this.args[1], 10) + 2; for (i = 2; i < keyStop; ++i) { + if (transform) { + this.args[i] = transform(this.args[i]); + } this._keys.push(this.args[i]); } break; case 'sort': + if (transform) { + this.args[0] = transform(this.args[0]); + } this._keys.push(this.args[0]); for (i = 1; i < this.args.length - 1; ++i) { if (typeof this.args[i] !== 'string') { @@ -106,22 +129,37 @@ Command.prototype.getKeys = function () { if (directive === 'GET') { i += 1; if (this.args[i] !== '#') { + if (transform) { + this.args[i] = transform(this.args[i]); + } this._keys.push(this.getKeyPart(this.args[i])); } } else if (directive === 'BY') { i += 1; + if (transform) { + this.args[i] = transform(this.args[i]); + } this._keys.push(this.getKeyPart(this.args[i])); } else if (directive === 'STORE') { i += 1; + if (transform) { + this.args[i] = transform(this.args[i]); + } this._keys.push(this.args[i]); } } break; case 'zunionstore': case 'zinterstore': + if (transform) { + this.args[0] = transform(this.args[0]); + } this._keys.push(this.args[0]); keyStop = parseInt(this.args[1], 10) + 2; for (i = 2; i < keyStop; ++i) { + if (transform) { + this.args[i] = transform(this.args[i]); + } this._keys.push(this.args[i]); } break; @@ -130,6 +168,9 @@ Command.prototype.getKeys = function () { keyStop = def.keyStop > 0 ? def.keyStop : this.args.length + def.keyStop + 1; if (keyStart >= 0 && keyStop <= this.args.length && keyStop > keyStart && def.step > 0) { for (i = keyStart; i < keyStop; i += def.step) { + if (transform) { + this.args[i] = transform(this.args[i]); + } this._keys.push(this.args[i]); } } diff --git a/lib/commander.js b/lib/commander.js index d2b19fc6..54828d06 100644 --- a/lib/commander.js +++ b/lib/commander.js @@ -65,7 +65,8 @@ Commander.prototype.send_command = Commander.prototype.call; * If omit, you have to pass the number of keys as the first argument every time you invoke the command */ Commander.prototype.defineCommand = function (name, definition) { - var script = new Script(definition.lua, definition.numberOfKeys); + var script = new Script(definition.lua, definition.numberOfKeys, + this.options.keyPrefix); this.scriptsSet[name] = script; this[name] = generateScriptingFunction(script, 'utf8'); this[name + 'Buffer'] = generateScriptingFunction(script, null); @@ -109,6 +110,9 @@ function generateFunction (_commandName, _encoding) { if (this.options.showFriendlyErrorStack) { options.errorStack = new Error().stack; } + if (this.options.keyPrefix) { + options.keyPrefix = this.options.keyPrefix; + } return this.sendCommand(new Command(commandName, args, options, callback)); }; diff --git a/lib/redis.js b/lib/redis.js index a5965ae4..8c9c5fd4 100644 --- a/lib/redis.js +++ b/lib/redis.js @@ -59,6 +59,7 @@ try { * When a new `Redis` instance is created, it will connect to Redis server automatically. * If you want to keep disconnected util a command is called, you can pass the `lazyConnect` option to * the constructor: + * @param {string} [options.keyPrefix=''] - The prefix to prepend to all keys in a command. * ```javascript * var redis = new Redis({ lazyConnect: true }); @@ -165,7 +166,8 @@ Redis.defaultOptions = { enableReadyCheck: true, autoResubscribe: true, autoResendUnfulfilledCommands: true, - lazyConnect: false + lazyConnect: false, + keyPrefix: '' }; Redis.prototype.resetCommandQueue = function () { diff --git a/lib/script.js b/lib/script.js index b9bdcf21..9ea60170 100644 --- a/lib/script.js +++ b/lib/script.js @@ -4,16 +4,20 @@ var Command = require('./command'); var crypto = require('crypto'); var Promise = require('bluebird'); -function Script(lua, numberOfKeys) { +function Script(lua, numberOfKeys, keyPrefix) { this.lua = lua; this.sha = crypto.createHash('sha1').update(this.lua).digest('hex'); this.numberOfKeys = typeof numberOfKeys === 'number' ? numberOfKeys : null; + this.keyPrefix = keyPrefix ? keyPrefix : ''; } Script.prototype.execute = function (container, args, options, callback) { if (typeof this.numberOfKeys === 'number') { args.unshift(this.numberOfKeys); } + if (this.keyPrefix) { + options.keyPrefix = this.keyPrefix; + } var evalsha = new Command('evalsha', [this.sha].concat(args), options); evalsha.isCustomCommand = true; diff --git a/test/functional/pipeline.js b/test/functional/pipeline.js index c4bc4905..45bd1715 100644 --- a/test/functional/pipeline.js +++ b/test/functional/pipeline.js @@ -72,6 +72,21 @@ describe('pipeline', function () { expect(pipeline.options).to.have.property('showFriendlyErrorStack', true); }); + it('should support key prefixing', function (done) { + var redis = new Redis({ keyPrefix: 'foo:' }); + redis.pipeline().set('bar', 'baz').get('bar').lpush('app1', 'test1').lpop('app1').keys('*').exec(function (err, results) { + expect(err).to.eql(null); + expect(results).to.eql([ + [null, 'OK'], + [null, 'baz'], + [null, 1], + [null, 'test1'], + [null, ['foo:bar']] + ]); + done(); + }); + }); + describe('#addBatch', function () { it('should accept commands in constructor', function (done) { var redis = new Redis(); diff --git a/test/functional/scripting.js b/test/functional/scripting.js index 3214389e..a1781352 100644 --- a/test/functional/scripting.js +++ b/test/functional/scripting.js @@ -182,4 +182,18 @@ describe('scripting', function () { }); }); }); + + it('should support key prefixing', function (done) { + var redis = new Redis({ keyPrefix: 'foo:' }); + + redis.defineCommand('echo', { + numberOfKeys: 2, + lua: 'return {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}' + }); + + redis.echo('k1', 'k2', 'a1', 'a2', function (err, result) { + expect(result).to.eql(['foo:k1', 'foo:k2', 'a1', 'a2']); + done(); + }); + }); }); diff --git a/test/functional/send_command.js b/test/functional/send_command.js index aec9c444..05d92168 100644 --- a/test/functional/send_command.js +++ b/test/functional/send_command.js @@ -116,4 +116,75 @@ describe('send command', function () { done(); }); }); + + it('should support key prefixing', function (done) { + var redis = new Redis({ keyPrefix: 'foo:' }); + redis.set('bar', 'baz'); + redis.get('bar', function (err, result) { + expect(result).to.eql('baz'); + redis.keys('*', function (err, result) { + expect(result).to.eql(['foo:bar']); + done(); + }); + }); + }); + + it('should support key prefixing with multiple keys', function (done) { + var redis = new Redis({ keyPrefix: 'foo:' }); + redis.lpush('app1', 'test1'); + redis.lpush('app2', 'test2'); + redis.lpush('app3', 'test3'); + redis.blpop('app1', 'app2', 'app3', 0, function (err, result) { + expect(result).to.eql(['foo:app1', 'test1']); + redis.keys('*', function (err, result) { + expect(result).to.have.members(['foo:app2', 'foo:app3']); + done(); + }); + }); + }); + + it('should support key prefixing for zunionstore', function (done) { + var redis = new Redis({ keyPrefix: 'foo:' }); + redis.zadd('zset1', 1, 'one'); + redis.zadd('zset1', 2, 'two'); + redis.zadd('zset2', 1, 'one'); + redis.zadd('zset2', 2, 'two'); + redis.zadd('zset2', 3, 'three'); + redis.zunionstore('out', 2, 'zset1', 'zset2', 'WEIGHTS', 2, 3, function (err, result) { + expect(result).to.eql(3); + redis.keys('*', function (err, result) { + expect(result).to.have.members(['foo:zset1', 'foo:zset2', 'foo:out']); + done(); + }); + }); + }); + + it('should support key prefixing for sort', function (done) { + var redis = new Redis({ keyPrefix: 'foo:' }); + redis.hset('object_1', 'name', 'better'); + redis.hset('weight_1', 'value', '20'); + redis.hset('object_2', 'name', 'best'); + redis.hset('weight_2', 'value', '30'); + redis.hset('object_3', 'name', 'good'); + redis.hset('weight_3', 'value', '10'); + redis.lpush('src', '1', '2', '3'); + redis.sort('src', 'BY', 'weight_*->value', 'GET', 'object_*->name', 'STORE', 'dest', function (err, result) { + redis.lrange('dest', 0, -1, function (err, result) { + expect(result).to.eql(['good', 'better', 'best']); + redis.keys('*', function (err, result) { + expect(result).to.have.members([ + 'foo:object_1', + 'foo:weight_1', + 'foo:object_2', + 'foo:weight_2', + 'foo:object_3', + 'foo:weight_3', + 'foo:src', + 'foo:dest' + ]); + done(); + }); + }); + }); + }); }); From 2cc2b39e752bd9de9afd3412728b9156d5044d64 Mon Sep 17 00:00:00 2001 From: Danny Guo Date: Wed, 22 Jul 2015 13:43:10 -0400 Subject: [PATCH 2/4] Don't check for transform everywhere when iterating over command keys --- lib/command.js | 37 +++++++++++++------------------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/lib/command.js b/lib/command.js index 5885cf04..cef954ae 100644 --- a/lib/command.js +++ b/lib/command.js @@ -101,6 +101,11 @@ Command.prototype.getKeys = function () { */ Command.prototype._iterateKeys = function (transform) { if (typeof this._keys === 'undefined') { + if (typeof transform !== 'function') { + transform = function (key) { + return key; + }; + } this._keys = []; var i, keyStart, keyStop; var def = commands[this.name]; @@ -110,16 +115,12 @@ Command.prototype._iterateKeys = function (transform) { case 'evalsha': keyStop = parseInt(this.args[1], 10) + 2; for (i = 2; i < keyStop; ++i) { - if (transform) { - this.args[i] = transform(this.args[i]); - } + this.args[i] = transform(this.args[i]); this._keys.push(this.args[i]); } break; case 'sort': - if (transform) { - this.args[0] = transform(this.args[0]); - } + this.args[0] = transform(this.args[0]); this._keys.push(this.args[0]); for (i = 1; i < this.args.length - 1; ++i) { if (typeof this.args[i] !== 'string') { @@ -129,37 +130,27 @@ Command.prototype._iterateKeys = function (transform) { if (directive === 'GET') { i += 1; if (this.args[i] !== '#') { - if (transform) { - this.args[i] = transform(this.args[i]); - } + this.args[i] = transform(this.args[i]); this._keys.push(this.getKeyPart(this.args[i])); } } else if (directive === 'BY') { i += 1; - if (transform) { - this.args[i] = transform(this.args[i]); - } + this.args[i] = transform(this.args[i]); this._keys.push(this.getKeyPart(this.args[i])); } else if (directive === 'STORE') { i += 1; - if (transform) { - this.args[i] = transform(this.args[i]); - } + this.args[i] = transform(this.args[i]); this._keys.push(this.args[i]); } } break; case 'zunionstore': case 'zinterstore': - if (transform) { - this.args[0] = transform(this.args[0]); - } + this.args[0] = transform(this.args[0]); this._keys.push(this.args[0]); keyStop = parseInt(this.args[1], 10) + 2; for (i = 2; i < keyStop; ++i) { - if (transform) { - this.args[i] = transform(this.args[i]); - } + this.args[i] = transform(this.args[i]); this._keys.push(this.args[i]); } break; @@ -168,9 +159,7 @@ Command.prototype._iterateKeys = function (transform) { keyStop = def.keyStop > 0 ? def.keyStop : this.args.length + def.keyStop + 1; if (keyStart >= 0 && keyStop <= this.args.length && keyStop > keyStart && def.step > 0) { for (i = keyStart; i < keyStop; i += def.step) { - if (transform) { - this.args[i] = transform(this.args[i]); - } + this.args[i] = transform(this.args[i]); this._keys.push(this.args[i]); } } From ae4dee570b9464d1567e3bda4a6f50896fce3f30 Mon Sep 17 00:00:00 2001 From: Danny Guo Date: Wed, 22 Jul 2015 13:46:35 -0400 Subject: [PATCH 3/4] Set options as empty object if it's undefined in the command constructor --- lib/command.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/command.js b/lib/command.js index cef954ae..eb5e68e0 100644 --- a/lib/command.js +++ b/lib/command.js @@ -39,11 +39,14 @@ var commands = require('../commands'); * @public */ function Command(name, args, options, callback) { + if (typeof options === 'undefined') { + options = {}; + } this.name = name; - this.replyEncoding = options && options.replyEncoding; - this.errorStack = options && options.errorStack; + this.replyEncoding = options.replyEncoding; + this.errorStack = options.errorStack; this.args = args ? _.flatten(args) : []; - if (options && options.keyPrefix) { + if (options.keyPrefix) { this._iterateKeys(function (key) { return options.keyPrefix + key; }); From 1a428563df463a35731b7dd5ec23b20f38b3837d Mon Sep 17 00:00:00 2001 From: Danny Guo Date: Wed, 22 Jul 2015 14:06:54 -0400 Subject: [PATCH 4/4] Store keyPrefix instead of looking it up in options every time --- lib/command.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/command.js b/lib/command.js index eb5e68e0..6bad49b0 100644 --- a/lib/command.js +++ b/lib/command.js @@ -46,9 +46,10 @@ function Command(name, args, options, callback) { this.replyEncoding = options.replyEncoding; this.errorStack = options.errorStack; this.args = args ? _.flatten(args) : []; - if (options.keyPrefix) { + var keyPrefix = options.keyPrefix; + if (keyPrefix) { this._iterateKeys(function (key) { - return options.keyPrefix + key; + return keyPrefix + key; }); } this.callback = callback;