From 7719809e3250c1bfd5e4fff4d141a20a2f3cf34c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20Wi=C5=9Bniowski?= Date: Tue, 10 Mar 2015 23:06:40 +0100 Subject: [PATCH 1/3] Added .jshintrc, modified Readme.md, mocha.opts and added some new dependencies to package.json --- .jshintrc | 4 ++++ Readme.md | 3 ++- package.json | 10 +++++++--- test/mocha.opts | 2 +- 4 files changed, 14 insertions(+), 5 deletions(-) create mode 100644 .jshintrc diff --git a/.jshintrc b/.jshintrc new file mode 100644 index 0000000..b6d6020 --- /dev/null +++ b/.jshintrc @@ -0,0 +1,4 @@ +{ + "node": true, + "laxcomma": true +} \ No newline at end of file diff --git a/Readme.md b/Readme.md index 6d35d57..0a4c2a8 100644 --- a/Readme.md +++ b/Readme.md @@ -425,7 +425,8 @@ _./test/auth.json_, which contains your credentials as JSON, for example: "secret": "", "bucket": "", "bucket2": "", - "bucketUsWest2": "" + "bucketUsWest2": "", + "bucketEuCentral1": "" } ``` diff --git a/package.json b/package.json index e5b6d13..9c89642 100644 --- a/package.json +++ b/package.json @@ -21,11 +21,15 @@ }, "bugs": "https://github.com/LearnBoost/knox/issues", "dependencies": { - "mime": "*", - "xml2js": "^0.4.4", + "aws4": "^0.4.2", "debug": "^1.0.2", + "duplexer": "^0.1.1", + "mime": "*", + "once": "^1.3.0", "stream-counter": "^1.0.0", - "once": "^1.3.0" + "through": "^2.3.6", + "xml2js": "^0.4.5", + "xmlbuilder": "^2.6.0" }, "devDependencies": { "mocha": "*" diff --git a/test/mocha.opts b/test/mocha.opts index cebbba1..a799972 100644 --- a/test/mocha.opts +++ b/test/mocha.opts @@ -1,4 +1,4 @@ --ui bdd --slow 1000ms ---timeout 5000ms +--timeout 20000ms --reporter spec From adb2da4111144a3e6a1e7fa58be95331002af17c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20Wi=C5=9Bniowski?= Date: Tue, 10 Mar 2015 23:08:45 +0100 Subject: [PATCH 2/3] Modified libraries to follow AWS Signature 4 specification --- lib/auth.js | 107 +------------------ lib/client.js | 277 +++++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 242 insertions(+), 142 deletions(-) diff --git a/lib/auth.js b/lib/auth.js index b1ffab1..a0ace97 100644 --- a/lib/auth.js +++ b/lib/auth.js @@ -11,44 +11,15 @@ */ var crypto = require('crypto') - , parse = require('url').parse; + , parse = require('url').parse + , aws4 = require('aws4'); /** * Query string params permitted in the canonicalized resource. * @see http://docs.amazonwebservices.com/AmazonS3/latest/dev/RESTAuthentication.html#ConstructingTheCanonicalizedResourceElement */ -var whitelist = [ - 'acl' - , 'delete' - , 'lifecycle' - , 'location' - , 'logging' - , 'notification' - , 'partNumber' - , 'policy' - , 'requestPayment' - , 'torrent' - , 'uploadId' - , 'uploads' - , 'versionId' - , 'versioning' - , 'versions' - , 'website' -]; - -/** - * Return an "Authorization" header value with the given `options` - * in the form of "AWS :" - * - * @param {Object} options - * @return {String} - * @api private - */ - -exports.authorization = function(options){ - return 'AWS ' + options.key + ':' + exports.sign(options); -}; +exports.sign = aws4.sign; /** * Simple HMAC-SHA1 Wrapper @@ -62,19 +33,6 @@ exports.hmacSha1 = function(options){ return crypto.createHmac('sha1', options.secret).update(new Buffer(options.message, 'utf-8')).digest('base64'); }; -/** - * Create a base64 sha1 HMAC for `options`. - * - * @param {Object} options - * @return {String} - * @api private - */ - -exports.sign = function(options){ - options.message = exports.stringToSign(options); - return exports.hmacSha1(options); -}; - /** * Create a base64 sha1 HMAC for `options`. * @@ -90,35 +48,6 @@ exports.signQuery = function(options){ return exports.hmacSha1(options); }; -/** - * Return a string for sign() with the given `options`. - * - * Spec: - * - * \n - * \n - * \n - * \n - * [headers\n] - * - * - * @param {Object} options - * @return {String} - * @api private - */ - -exports.stringToSign = function(options){ - var headers = options.amazonHeaders || ''; - if (headers) headers += '\n'; - return [ - options.verb - , options.md5 - , options.contentType - , options.date instanceof Date ? options.date.toUTCString() : options.date - , headers + options.resource - ].join('\n'); -}; - /** * Return a string for sign() with the given `options`, but is meant exclusively * for S3 presigned URLs @@ -186,32 +115,4 @@ exports.canonicalizeHeaders = function(headers){ return a > b ? 1 : -1; } return buf.sort(headerSort).join('\n'); -}; - -/** - * Perform the following: - * - * - ignore non sub-resources - * - sort lexicographically - * - * @param {String} a URI-encoded resource (path + query string) - * @return {String} - * @api private - */ - -exports.canonicalizeResource = function(resource){ - var url = parse(resource, true) - , path = url.pathname - , buf = []; - - // apply the query string whitelist - Object.keys(url.query).forEach(function (key) { - if (whitelist.indexOf(key) != -1) { - buf.push(key + (url.query[key] ? "=" + url.query[key] : '')); - } - }); - - return path + (buf.length - ? '?' + buf.sort().join('&') - : ''); -}; +}; \ No newline at end of file diff --git a/lib/client.js b/lib/client.js index ea2b3ed..29507ec 100644 --- a/lib/client.js +++ b/lib/client.js @@ -23,7 +23,12 @@ var Emitter = require('events').EventEmitter , once = require('once') , xml2js = require('xml2js') , StreamCounter = require('stream-counter') - , qs = require('querystring'); + , qs = require('querystring') + , duplexer = require('duplexer') + , through = require('through') + , xmlBuilder = require('xmlbuilder') + , url = require('url'); + // The max for multi-object delete, bucket listings, etc. var BUCKET_OPS_MAX = 1000; @@ -35,6 +40,7 @@ var MAX_US_STANDARD_BUCKET_LENGTH = 255; var US_STANDARD_BUCKET = /^[A-Za-z0-9\._-]*$/; var BUCKET_LABEL = /^(?:[a-z0-9][a-z0-9-]*[a-z0-9]|[a-z0-9])$/; var IPV4_ADDRESS = /^(\d{1,3}\.){3}(\d{1,3})$/; +var MIN_UPLOAD_CHUNK_SIZE = 5 * 1028 * 1028; /** * Register event listeners on a request object to convert standard http @@ -61,6 +67,12 @@ function removeLeadingSlash(filename) { return filename[0] === '/' ? filename.substring(1) : filename; } +function fixQueryString(filename) { + var parsedFilename = url.parse(filename); + parsedFilename.search = "?" + qs.stringify(qs.parse(parsedFilename.query)); + return url.format(parsedFilename); +} + function encodeSpecialCharacters(filename) { // Note: these characters are valid in URIs, but S3 does not like them for // some reason. @@ -203,12 +215,12 @@ var Client = module.exports = exports = function Client(options) { if (!options.endpoint) { if (!options.region || options.region === 'us-standard' || options.region === 'us-east-1') { options.endpoint = 's3.amazonaws.com'; - options.region = 'us-standard'; + options.region = 'us-east-1'; } else { options.endpoint = 's3-' + options.region + '.amazonaws.com'; } - if (options.region !== 'us-standard') { + if (options.region !== 'us-east-1') { if (dnsUncompliance) { throw new Error('Outside of the us-standard region, bucket names must' + ' be DNS-compliant. The name "' + options.bucket + @@ -247,48 +259,236 @@ var Client = module.exports = exports = function Client(options) { * @param {String} method * @param {String} filename * @param {Object} headers + * @param {Buffer|string} body Optional body for sending delete requests or other non-multipart uploads * @return {ClientRequest} * @api private */ -Client.prototype.request = function(method, filename, headers){ - var options = { hostname: this.host, agent: this.agent, port: this.port } - , date = new Date - , headers = headers || {} - , fixedFilename = ensureLeadingSlash(filename); +Client.prototype.request = function(method, filename, headers, body){ + var fixedFilename = fixQueryString(ensureLeadingSlash(filename)); + + headers = headers || {}; + + if(body) { + headers['Content-Length'] = body.length; + headers['Content-MD5'] = crypto.createHash('md5').update(body).digest('base64'); + } + + var options = this.signRequest(headers, method, fixedFilename, body); + + var req = (this.secure ? https : http).request(options); + + req.url = this.url(filename); + + debug('%s %s', method, req.url); + + if(body) { + req.end(body); + } + + return req; +}; + +/** + * Singing request method + * + * @param {Object} headers + * @param {String} method + * @param {String} path + * @param {Buffer|String} body + * @return {Object} Option object suitable for `http(s).request` method + * @api private + */ + +Client.prototype.signRequest = function(headers, method, path, body) { + + headers = headers || {}; - // Default headers - headers.Date = date.toUTCString() if (this.style === 'virtualHosted') { headers.Host = this.host; } - if ('undefined' != typeof this.token) - headers['x-amz-security-token'] = this.token; + var pathPrefix = this.style === 'path' ? '/' + this.bucket : ''; - // Authorization header - headers.Authorization = auth.authorization({ - key: this.key - , secret: this.secret - , verb: method - , date: date - , resource: auth.canonicalizeResource('/' + this.bucket + fixedFilename) - , contentType: getHeader(headers, 'content-type') - , md5: getHeader(headers, 'content-md5') || '' - , amazonHeaders: auth.canonicalizeHeaders(headers) + return auth.sign({ + service: 's3' + , region: this.region + , host: this.host + , agent: this.agent + , port: this.port || null + , method: method + , path: pathPrefix + path + , headers: headers + , body: body || null + }, { + accessKeyId: this.key || null + , secretAccessKey: this.secret || null + , sessionToken: this.token || null }); +}; - var pathPrefix = this.style === 'path' ? '/' + this.bucket : ''; +/** + * Multipart upload with `filename` and optional `headers` and `chunkSize`. + * + * @param {String} filename + * @param {Object} headers + * @param {Number} chunkSize + * @return {DuplexStream} + * @api private + */ - // Issue request - options.method = method; - options.path = pathPrefix + fixedFilename; - options.headers = headers; - var req = (this.secure ? https : http).request(options); - req.url = this.url(filename); - debug('%s %s', method, req.url); +Client.prototype.createMultipartUploadStream = function(filename, headers, chunkSize) { - return req; + var duplexStream + , bufferPartStream + , multiPartStream + , finalizeUploadStream; + + var fixedFilename = encodeSpecialCharacters(ensureLeadingSlash(filename)) + , initChunks = new Buffer(0) + , bodyChunks = new Buffer(0) + , partNumber = 0 + , uploadId = null + , initialHeaders = {}; + + headers = headers || {}; + + if(!chunkSize) { + chunkSize = MIN_UPLOAD_CHUNK_SIZE; + } else { + chunkSize = Math.max(chunkSize, MIN_UPLOAD_CHUNK_SIZE); + } + + if(headers['Content-Type']) { + initialHeaders['Content-Type'] = headers['Content-Type']; + } + + // Upload Initialization - This request gets the UploadID that is needed for uploaded parts + + var req = this.request('POST', fixedFilename + '?uploads=', initialHeaders); + + req.on('response', function(res) { + res.on('data', function(chunk) { + if(!(chunk instanceof Buffer)) { + chunk = new Buffer(chunk, 'utf8'); + } + initChunks = Buffer.concat([initChunks, chunk]); + }); + res.on('end', function() { + xml2js.parseString(initChunks.toString('utf8'), function (err, result) { + uploadId = result.InitiateMultipartUploadResult.UploadId[0]; + bufferPartStream.resume(); + }); + }); + }); + + // Paused buffer stream - Buffers chunks up chunkSize or minimum of MIN_UPLOAD_CHUNK_SIZE paused untill + // the initial request for UploadId is made + + bufferPartStream = through(function(chunk) { + if(!(chunk instanceof Buffer)) { + chunk = new Buffer(chunk, 'utf8'); + } + bodyChunks = Buffer.concat([bodyChunks, chunk]); + if(bodyChunks.length >= chunkSize) { + this.queue(bodyChunks); + bodyChunks = new Buffer(0); + } + }, function() { + this.queue(bodyChunks); + bodyChunks = new Buffer(0); + this.queue(null); + }); + + bufferPartStream.pause(); + + // Multipart stream - receives buffered chunks and creates 1 requests for each buffered chunk. + + var that = this; + + multiPartStream = through(function(chunk) { + partNumber += 1; + this.pause(); + + that.uploadPart('PUT', fixedFilename, partNumber, uploadId, null, bodyChunks, function(res) { + this.queue([partNumber,res.headers.etag.replace(/^"|"$/g, '')]); + this.resume(); + }.bind(this)); + }); + + // Finalize stream - gathers pairs of partNumber and etag and sends them as a finialization + // of this particular upload + + finalizeUploadStream = through(function(chunk) { + this.parts = this.parts || {}; + this.parts[chunk[0]] = chunk[1]; + }, function() { + + var parts = []; + + for(var partNumber in this.parts) { + parts.push({ + Part: { + PartNumber: partNumber, + ETag: {'#text': this.parts[partNumber]} + } + }); + } + + var xml = xmlBuilder.create({CompleteMultipartUpload: parts}); + + var headers = { + 'Content-Type': 'application/xml' + }; + + that.uploadPart('POST', fixedFilename, null, uploadId, headers, xml.toString(), function(res) { + duplexStream.emit('response', res); + res.on('data', function(chunk) { + this.queue(chunk); + }.bind(this)); + res.on('end', function() { + this.queue(null); + }.bind(this)); + }.bind(this)); + }); + + bufferPartStream.pipe(multiPartStream).pipe(finalizeUploadStream); + + // We're returning duplex stream so we can stream data + // to `bufferPartStream` and read `data` from the `finalizeUploadStream` + + duplexStream = duplexer(bufferPartStream, finalizeUploadStream); + duplexStream.url = this.url(fixedFilename); + + req.end(); + + return duplexStream; +}; + +/** + * Single part upload + * + * @param {String} method It's usually `PUT` when uploading a part or `POST` when finalizing + * @param {String} fixedFilename + * @param {Number} partNumber + * @param {String} uploadId + * @param {Object} headers + * @param {Buffer|String} buffer + * @param {Function} callback + * @return {DuplexStream} + * + * @api private + */ + +Client.prototype.uploadPart = function(method, fixedFilename, partNumber, uploadId, headers, buffer, callback) { + + headers = headers || {}; + + var req = this.request(method, fixedFilename + '?' + (partNumber ? 'partNumber=' + partNumber + '&' : '') + 'uploadId=' + uploadId , headers, buffer); + + req.on('response', function(res) { + callback(res); + }); }; /** @@ -323,7 +523,8 @@ Client.prototype.request = function(method, filename, headers){ Client.prototype.put = function(filename, headers){ headers = utils.merge({}, headers || {}); - return this.request('PUT', encodeSpecialCharacters(filename), headers); + + return this.createMultipartUploadStream(encodeSpecialCharacters(filename), headers); }; /** @@ -654,7 +855,7 @@ function xmlEscape(string) { function makeDeleteXmlBuffer(keys) { var tags = keys.map(function(key){ return '' + - xmlEscape(removeLeadingSlash(key)) + + xmlEscape(encodeSpecialCharacters(removeLeadingSlash(key))) + ''; }); return new Buffer('' + @@ -684,13 +885,11 @@ Client.prototype.deleteMultiple = function(filenames, headers, fn){ var xml = makeDeleteXmlBuffer(filenames); - headers['Content-Length'] = xml.length; - headers['Content-MD5'] = crypto.createHash('md5').update(xml).digest('base64'); + headers['Content-Type'] = 'application/xml'; - var req = this.request('POST', '/?delete', headers); + var req = this.request('POST', '/?delete=', headers, xml); fn = once(fn); registerReqListeners(req, fn); - req.end(xml); return req; }; @@ -823,7 +1022,7 @@ Client.prototype.list = function(params, headers, fn){ */ Client.prototype.http = function(filename){ - filename = encodeSpecialCharacters(ensureLeadingSlash(filename)); + filename = ensureLeadingSlash(filename); return 'http://' + this.urlBase + filename; }; @@ -837,7 +1036,7 @@ Client.prototype.http = function(filename){ */ Client.prototype.https = function(filename){ - filename = encodeSpecialCharacters(ensureLeadingSlash(filename)); + filename = ensureLeadingSlash(filename); return 'https://' + this.urlBase + filename; }; From ec00d79457caa7c738313d3969a79a8521183122 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marcin=20Wi=C5=9Bniowski?= Date: Tue, 10 Mar 2015 23:09:53 +0100 Subject: [PATCH 3/3] Modified tests to reflect changes to the lib directory --- test/auth.test.js | 168 -------------------------------------- test/createClient.test.js | 32 ++++---- test/initClients.js | 15 +++- test/knox.test.js | 56 +++++++++---- 4 files changed, 67 insertions(+), 204 deletions(-) diff --git a/test/auth.test.js b/test/auth.test.js index aab2b0b..4c14d26 100644 --- a/test/auth.test.js +++ b/test/auth.test.js @@ -5,144 +5,6 @@ var knox = require('..') , assert = require('assert'); describe('knox.auth', function () { - describe('.stringToSign()', function () { - specify('for a basic PUT', function () { - var actual = auth.stringToSign({ - verb: 'PUT' - , md5: '09c68b914d66457508f6ad727d860d5b' - , contentType: 'text/plain' - , resource: '/learnboost' - , date: new Date('Mon, May 25 1987 00:00:00 GMT') - }); - - var expected = [ - 'PUT' - , '09c68b914d66457508f6ad727d860d5b' - , 'text/plain' - , new Date('Mon, May 25 1987 00:00:00 GMT').toUTCString() - , '/learnboost' - ].join('\n'); - - assert.equal(actual, expected); - }); - }); - - describe('.sign()', function () { - specify('for a basic PUT', function () { - var actual = auth.sign({ - verb: 'PUT' - , secret: 'test' - , md5: '09c68b914d66457508f6ad727d860d5b' - , contentType: 'text/plain' - , resource: '/learnboost' - , date: new Date('Mon, May 25 1987 00:00:00 GMT') - }); - - assert.equal(actual, '7xIdjyy+W17/k0le5kwBnfrZTiM='); - }); - }); - - describe('.authorization() [from the Amazon docs]', function () { - // http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#RESTAuthenticationExamples - - specify('Example Object GET', function () { - var actual = auth.authorization({ - verb: 'GET' - , key: 'AKIAIOSFODNN7EXAMPLE' - , secret: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' - , resource: auth.canonicalizeResource('/johnsmith/photos/puppy.jpg') - , date: 'Tue, 27 Mar 2007 19:36:42 +0000' - }); - - assert.equal(actual, 'AWS AKIAIOSFODNN7EXAMPLE:bWq2s1WEIj+Ydj0vQ697zp+IXMU='); - }); - - specify('Example Object PUT', function () { - var actual = auth.authorization({ - verb: 'PUT' - , key: 'AKIAIOSFODNN7EXAMPLE' - , secret: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' - , resource: auth.canonicalizeResource('/johnsmith/photos/puppy.jpg') - , contentType: 'image/jpeg' - , date: 'Tue, 27 Mar 2007 21:15:45 +0000' - }); - - assert.equal(actual, 'AWS AKIAIOSFODNN7EXAMPLE:MyyxeRY7whkBe+bq8fHCL/2kKUg='); - }); - - specify('Example List', function () { - var actual = auth.authorization({ - verb: 'GET' - , key: 'AKIAIOSFODNN7EXAMPLE' - , secret: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' - , resource: auth.canonicalizeResource('/johnsmith/') - , date: 'Tue, 27 Mar 2007 19:42:41 +0000' - }); - - assert.equal(actual, 'AWS AKIAIOSFODNN7EXAMPLE:htDYFYduRNen8P9ZfE/s9SuKy0U='); - }); - - specify('Example Fetch', function () { - var actual = auth.authorization({ - verb: 'GET' - , key: 'AKIAIOSFODNN7EXAMPLE' - , secret: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' - , resource: auth.canonicalizeResource('/johnsmith/?acl') - , date: 'Tue, 27 Mar 2007 19:44:46 +0000' - }); - - assert.equal(actual, 'AWS AKIAIOSFODNN7EXAMPLE:c2WLPFtWHVgbEmeEG93a4cG37dM='); - }); - - specify('Example Delete', function () { - // This is modified from the docs: knox does not allow setting the date - // through x-amz-date, so we test that the x-amz-date is ignored and the - // date is instead used. - var actual = auth.authorization({ - verb: 'DELETE' - , key: 'AKIAIOSFODNN7EXAMPLE' - , secret: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' - , resource: auth.canonicalizeResource('/johnsmith/photos/puppy.jpg') - , amazonHeaders: auth.canonicalizeHeaders({ - 'x-amz-date': 'Tue, 27 Mar 2007 21:20:27 +0000' - }) - , date: 'Tue, 27 Mar 2007 21:20:26 +0000' - }); - - assert.equal(actual, 'AWS AKIAIOSFODNN7EXAMPLE:lx3byBScXR6KzyMaifNkardMwNk='); - }); - - specify.skip('Example Upload', function () { - // Knox doesn't support multiple values for a single header; see - // discussion at https://github.com/LearnBoost/knox/pull/6. Elegant pull - // requests to implement this feature welcome! - }); - - specify('Example List All My Buckets', function () { - var actual = auth.authorization({ - verb: 'GET' - , key: 'AKIAIOSFODNN7EXAMPLE' - , secret: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' - , resource: auth.canonicalizeResource('/') - , date: 'Wed, 28 Mar 2007 01:29:59 +0000' - }); - - assert.equal(actual, 'AWS AKIAIOSFODNN7EXAMPLE:qGdzdERIC03wnaRNKh6OqZehG9s='); - }); - - specify('Example Unicode Keys', function () { - var actual = auth.authorization({ - verb: 'GET' - , key: 'AKIAIOSFODNN7EXAMPLE' - , secret: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' - , resource: auth.canonicalizeResource('/dictionary/fran%C3%A7ais/pr%c3%a9f%c3%a8re') - , date: 'Wed, 28 Mar 2007 01:49:49 +0000' - }); - - assert.equal(actual, 'AWS AKIAIOSFODNN7EXAMPLE:DNEZGsoieTZ92F3bUfSPQcbGmlM='); - }); - }); - describe('.canonicalizeHeaders()', function () { specify('with no Amazon headers', function () { assert.equal(auth.canonicalizeHeaders({}), ''); @@ -221,34 +83,4 @@ describe('knox.auth', function () { assert.equal(actual, expected); }); }); - - describe('.canonicalizeResource()', function () { - specify('for a bucket alone', function () { - assert.equal(auth.canonicalizeResource('/bucket/'), '/bucket/'); - }); - - specify('for a bucket, folder, and file', function () { - assert.equal(auth.canonicalizeResource('/bucket/test/user2.json'), '/bucket/test/user2.json'); - }); - - specify('for a bucket\'s ACL list URL', function () { - assert.equal(auth.canonicalizeResource('/bucket/?acl'), '/bucket/?acl'); - }); - - specify('for a bucket\'s delete multiple URL', function () { - assert.equal(auth.canonicalizeResource('/bucket/?delete'), '/bucket/?delete'); - }); - - specify('for a bucket filtered by a simple prefix', function () { - assert.equal(auth.canonicalizeResource('/bucket/?prefix=logs'), '/bucket/'); - }); - - specify('for a bucket filtered by a simple prefix and a delimiter', function () { - assert.equal(auth.canonicalizeResource('/bucket/?prefix=logs/&delimiter=/'), '/bucket/'); - }); - - specify('for a bucket filtered by a complex prefix and a delimiter', function () { - assert.equal(auth.canonicalizeResource('/bucket/?prefix=log%20files/&delimiter=/'), '/bucket/'); - }); - }); }); diff --git a/test/createClient.test.js b/test/createClient.test.js index d57a0e9..1a1aa5d 100644 --- a/test/createClient.test.js +++ b/test/createClient.test.js @@ -61,7 +61,7 @@ describe('knox.createClient()', function () { }); describe('bucket names', function () { - describe('in us-standard region', function () { + describe('in us-east-1 region', function () { it('should throw when bucket names are too short', function () { assert.throws( function () { @@ -207,7 +207,7 @@ describe('knox.createClient()', function () { assert.equal(client.secure, true); assert.equal(client.style, 'virtualHosted'); - assert.equal(client.region, 'us-standard'); + assert.equal(client.region, 'us-east-1'); assert.equal(client.endpoint, 's3.amazonaws.com'); assert.equal(client.url('file'), 'https://misc.s3.amazonaws.com/file'); }); @@ -245,18 +245,18 @@ describe('knox.createClient()', function () { assert.equal(client.url('file'), 'https://misc.s3-us-west-1.amazonaws.com/file'); }); - it('should derive endpoint correctly from explicit us-standard region', function () { + it('should derive endpoint correctly from explicit us-east-1 region', function () { var client = knox.createClient({ key: 'foobar' , secret: 'baz' , bucket: 'misc' , style: 'virtualHosted' - , region: 'us-standard' + , region: 'us-east-1' }); assert.equal(client.secure, true); assert.equal(client.style, 'virtualHosted'); - assert.equal(client.region, 'us-standard'); + assert.equal(client.region, 'us-east-1'); assert.equal(client.endpoint, 's3.amazonaws.com'); assert.equal(client.url('file'), 'https://misc.s3.amazonaws.com/file'); }); @@ -272,7 +272,7 @@ describe('knox.createClient()', function () { assert.equal(client.secure, true); assert.equal(client.style, 'virtualHosted'); - assert.equal(client.region, 'us-standard'); + assert.equal(client.region, 'us-east-1'); assert.equal(client.endpoint, 's3.amazonaws.com'); assert.equal(client.url('file'), 'https://misc.s3.amazonaws.com/file'); }); @@ -324,7 +324,7 @@ describe('knox.createClient()', function () { assert.equal(client.secure, true); assert.equal(client.style, 'path'); - assert.equal(client.region, 'us-standard'); + assert.equal(client.region, 'us-east-1'); assert.equal(client.endpoint, 's3.amazonaws.com'); assert.equal(client.url('file'), 'https://s3.amazonaws.com/misc/file'); }); @@ -362,18 +362,18 @@ describe('knox.createClient()', function () { assert.equal(client.url('file'), 'https://s3-us-west-1.amazonaws.com/misc/file'); }); - it('should derive endpoint correctly from explicit us-standard region', function () { + it('should derive endpoint correctly from explicit us-east-1 region', function () { var client = knox.createClient({ key: 'foobar' , secret: 'baz' , bucket: 'misc' , style: 'path' - , region: 'us-standard' + , region: 'us-east-1' }); assert.equal(client.secure, true); assert.equal(client.style, 'path'); - assert.equal(client.region, 'us-standard'); + assert.equal(client.region, 'us-east-1'); assert.equal(client.endpoint, 's3.amazonaws.com'); assert.equal(client.url('file'), 'https://s3.amazonaws.com/misc/file'); }); @@ -424,7 +424,7 @@ describe('knox.createClient()', function () { assert.equal(client.secure, true); assert.equal(client.style, 'virtualHosted'); - assert.equal(client.region, 'us-standard'); + assert.equal(client.region, 'us-east-1'); assert.equal(client.endpoint, 's3.amazonaws.com'); assert.equal(client.url('file'), 'https://misc.s3.amazonaws.com/file'); }); @@ -438,7 +438,7 @@ describe('knox.createClient()', function () { assert.equal(client.secure, true); assert.equal(client.style, 'path'); - assert.equal(client.region, 'us-standard'); + assert.equal(client.region, 'us-east-1'); assert.equal(client.endpoint, 's3.amazonaws.com'); assert.equal(client.url('file'), 'https://s3.amazonaws.com/misc.bucket/file'); }); @@ -452,7 +452,7 @@ describe('knox.createClient()', function () { assert.equal(client.secure, true); assert.equal(client.style, 'path'); - assert.equal(client.region, 'us-standard'); + assert.equal(client.region, 'us-east-1'); assert.equal(client.endpoint, 's3.amazonaws.com'); assert.equal(client.url('file'), 'https://s3.amazonaws.com/MiscBucket/file'); }); @@ -466,7 +466,7 @@ describe('knox.createClient()', function () { assert.equal(client.secure, true); assert.equal(client.style, 'path'); - assert.equal(client.region, 'us-standard'); + assert.equal(client.region, 'us-east-1'); assert.equal(client.endpoint, 's3.amazonaws.com'); assert.equal(client.url('file'), 'https://s3.amazonaws.com/misc_bucket/file'); }); @@ -480,7 +480,7 @@ describe('knox.createClient()', function () { assert.equal(client.secure, true); assert.equal(client.style, 'path'); - assert.equal(client.region, 'us-standard'); + assert.equal(client.region, 'us-east-1'); assert.equal(client.endpoint, 's3.amazonaws.com'); assert.equal(client.url('file'), 'https://s3.amazonaws.com/-bucket/file'); }); @@ -496,7 +496,7 @@ describe('knox.createClient()', function () { assert.equal(client.secure, false); assert.equal(client.style, 'virtualHosted'); - assert.equal(client.region, 'us-standard'); + assert.equal(client.region, 'us-east-1'); assert.equal(client.endpoint, 's3.amazonaws.com'); assert.equal(client.url('file'), 'http://misc.bucket.s3.amazonaws.com/file'); }); diff --git a/test/initClients.js b/test/initClients.js index 048b314..3e5ab2e 100644 --- a/test/initClients.js +++ b/test/initClients.js @@ -5,7 +5,7 @@ var knox = require('..') , assert = require('assert'); module.exports = function(style){ - var client, client2, clientUsWest2; + var client, client2, clientUsWest2, clientEuFrankfurt; try { var auth = utils.merge({ style: style }, require('./auth.json')); @@ -13,9 +13,13 @@ module.exports = function(style){ assert(auth.bucket, 'bucket must exist'); assert(auth.bucket2, 'bucket2 must exist'); assert(auth.bucketUsWest2, 'bucketUsWest2 must exist'); + assert(auth.bucketEuCentral1, 'bucketEuCentral1 must exist'); assert.notEqual(auth.bucket, auth.bucket2, 'bucket should not equal bucket2.'); assert.notEqual(auth.bucket, auth.bucketUsWest2, 'bucket should not equal bucketUsWest2.'); assert.notEqual(auth.bucket2, auth.bucketUsWest2, 'bucket2 should not equal bucketUsWest2.'); + assert.notEqual(auth.bucket, auth.bucketEuCentral1, 'bucket should not equal bucketEuCentral1.'); + assert.notEqual(auth.bucket2, auth.bucketEuCentral1, 'bucket2 should not equal bucketEuCentral1.'); + assert.notEqual(auth.bucketUsWest2, auth.bucketEuCentral1, 'bucketUsWest2 should not equal bucketEuCentral1.'); var auth1 = utils.merge({}, auth); client = knox.createClient(auth1); @@ -26,13 +30,20 @@ module.exports = function(style){ var authUsWest2 = utils.merge({}, auth); authUsWest2.bucket = auth.bucketUsWest2; + + var authEuFrankfurt = utils.merge({}, auth); + authEuFrankfurt.bucket = auth.bucketEuCentral1; + // Without this we get a 307 redirect // that putFile can't handle (issue #66). Later // when there is an implementation of #66 we can test // both with and without this option present, but it's // always a good idea for performance authUsWest2.region = 'us-west-2'; + authEuFrankfurt.region = 'eu-central-1'; + clientUsWest2 = knox.createClient(authUsWest2); + clientEuFrankfurt = knox.createClient(authUsWest2); } catch (err) { console.error(err); console.error('The tests require test/auth.json to contain JSON with ' + @@ -44,5 +55,5 @@ module.exports = function(style){ process.exit(1); } - return { client: client, client2: client2, clientUsWest2: clientUsWest2 }; + return { client: client, client2: client2, clientUsWest2: clientUsWest2, clientEuFrankfurt: clientEuFrankfurt }; }; diff --git a/test/knox.test.js b/test/knox.test.js index 0b5ee8e..ff538ae 100644 --- a/test/knox.test.js +++ b/test/knox.test.js @@ -18,6 +18,7 @@ function runTestsForStyle(style, userFriendlyName) { var client = clients.client; var client2 = clients.client2; var clientUsWest2 = clients.clientUsWest2; + var clientEuFrankfurt = clients.clientEuFrankfurt; describe('put()', function () { specify('from a file statted and read into a buffer', function (done) { @@ -120,16 +121,6 @@ function runTestsForStyle(style, userFriendlyName) { req.end(data); }); - - it('should lower-case headers on requests', function () { - var headers = { 'X-Amz-Acl': 'private' }; - var req = client.put('/test/user.json', headers); - - assert.equal(req.getHeader('x-amz-acl'), 'private'); - - req.on('error', function (){}); // swallow "socket hang up" from aborting - req.abort(); - }); }); describe('putStream()', function () { @@ -244,6 +235,18 @@ function runTestsForStyle(style, userFriendlyName) { }); }); + it('should work the same in eu-central-1', function (done) { + clientEuFrankfurt.putFile(jsonFixture, '/test/user2.json', function (err, res) { + assert.ifError(err); + assert.equal(res.statusCode, 200); + + clientEuFrankfurt.get('/test/user2.json').on('response', function (res) { + assert.equal(res.headers['content-type'], 'application/json'); + done(); + }).end(); + }); + }); + it('should emit "progress" events', function (done) { var progressHappened = false; @@ -251,7 +254,7 @@ function runTestsForStyle(style, userFriendlyName) { assert.ifError(err); assert.equal(res.statusCode, 200); - clientUsWest2.get('/test/user2.json').on('response', function (res) { + client.get('/test/user2.json').on('response', function (res) { assert.equal(res.headers['content-type'], 'application/json'); assert(progressHappened); done(); @@ -281,7 +284,7 @@ function runTestsForStyle(style, userFriendlyName) { specify('with a lower-case "content-type" header', function (done) { var buffer = new Buffer('a string of stuff'); - var headers = { 'content-type': 'text/plain' }; + var headers = { 'Content-Type': 'text/plain' }; client.putBuffer(buffer, '/buffer2.txt', headers, function (err, res) { assert.ifError(err); @@ -568,16 +571,13 @@ function runTestsForStyle(style, userFriendlyName) { ''; var req = client.request('POST', '/?delete', { - 'Content-Length': xml.length, - 'Content-MD5': crypto.createHash('md5').update(xml).digest('base64'), - 'Accept:': '*\/*' - }) + 'Content-Type': 'application/xml' + }, xml) .on('error',done) .on('response', function (res) { assert.equal(res.statusCode, 200); done(); - }) - .end(xml); + }); }); }); @@ -607,6 +607,7 @@ function runTestsForStyle(style, userFriendlyName) { 'google', 'buffer with spaces.txt', 'buffer+with+pluses.txt', 'buffer?with?questions.txt', '/ø', 'ø/ø', '/test/versioned.txt#1']; client.deleteMultiple(files, function (err, res) { + assert.ifError(err); assert.equal(res.statusCode, 200); @@ -647,6 +648,14 @@ function runTestsForStyle(style, userFriendlyName) { done(); }); }); + + it('should work in bucketEuCentral1', function (done) { + clientEuFrankfurt.deleteMultiple(['test/user2.json'], function (err, res) { + assert.ifError(err); + assert.equal(res.statusCode, 200); + done(); + }); + }); }); describe('we should clean up and not use people\'s S3 $$$', function () { @@ -682,6 +691,17 @@ function runTestsForStyle(style, userFriendlyName) { var keys = data.Contents.map(function (entry) { return entry.Key; }); assert.deepEqual(keys, []); + done(); + }); + }); + specify('in bucketEuCentral1', function (done) { + clientEuFrankfurt.list(function (err, data) { + assert.ifError(err); + + // Do the assertion like this for nicer error reporting. + var keys = data.Contents.map(function (entry) { return entry.Key; }); + assert.deepEqual(keys, []); + done(); }); });