diff --git a/config/index.js b/config/index.js index 6ed0c5619..14c63924b 100644 --- a/config/index.js +++ b/config/index.js @@ -207,6 +207,14 @@ var config = { // sync interval, default is 10 minutes syncInterval: '10m', + // sync polular modules, default to false + // because cnpm can't auto sync tag change for now + // so we want to sync popular modules to ensure their tags + syncPopular: false, + syncPopularInterval: '1h', + // top 100 + topPopular: 100, + // sync devDependencies or not, default is false syncDevDependencies: false, diff --git a/controllers/registry/module.js b/controllers/registry/module.js index 48b304216..3360643bc 100644 --- a/controllers/registry/module.js +++ b/controllers/registry/module.js @@ -26,7 +26,6 @@ var coWrite = require('co-write'); var urlparse = require('url').parse; var mime = require('mime'); var semver = require('semver'); -var ms = require('ms'); var config = require('../../config'); var Module = require('../../proxy/module'); var Total = require('../../proxy/total'); diff --git a/controllers/utils.js b/controllers/utils.js index 1ce773511..393fe8f78 100644 --- a/controllers/utils.js +++ b/controllers/utils.js @@ -18,7 +18,7 @@ var debug = require('debug')('cnpmjs.org:controllers:utils'); var path = require('path'); var fs = require('fs'); var utility = require('utility'); -var ms = require('ms'); +var ms = require('humanize-ms'); var nfs = require('../common/nfs'); var config = require('../config'); diff --git a/package.json b/package.json index 3c79ef05e..c594d7136 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "giturl": "~0.0.3", "graceful": "~0.1.0", "gravatar": "~1.1.0", + "humanize-ms": "~1.0.0", "humanize-number": "~0.0.2", "koa": "~0.12.2", "koa-limit": "~1.0.2", @@ -40,9 +41,8 @@ "mini-logger": "~1.0.0", "mkdirp": "~0.5.0", "moment": "~2.8.3", - "ms": "~0.6.2", "multiline": "~1.0.1", - "mysql": "~2.5.1", + "mysql": "~2.5.2", "nodemailer": "~1.3.0", "ready": "~0.1.1", "redis": "~0.12.1", @@ -59,7 +59,7 @@ "istanbul-harmony": "~0.3.0", "jshint": "~2.5.6", "mm": "~0.2.1", - "mocha": "~1.21.4", + "mocha": "~1.21.5", "node-dev": "*", "pedding": "~1.0.0", "should": "~4.0.4", diff --git a/proxy/npm.js b/proxy/npm.js index 9f8d8053d..03fbf95f0 100644 --- a/proxy/npm.js +++ b/proxy/npm.js @@ -64,17 +64,38 @@ exports.get = function *(name) { return data; }; -exports.getAllSince = function *(startkey) { +exports.getAllSince = function *(startkey, timeout) { var r = yield *request('/-/all/since?stale=update_after&startkey=' + startkey, { - timeout: 300000 + timeout: timeout || 300000 }); return r.data; }; -exports.getShort = function *() { +exports.getShort = function *(timeout) { var r = yield *request('/-/short', { - timeout: 300000, + timeout: timeout || 300000, registry: 'http://r.cnpmjs.org', // registry.npmjs.org/-/short is 404 now. }); return r.data; }; + +exports.getPopular = function* (top, timeout) { + var r = yield *request('/-/_view/dependedUpon?group_level=1', { + registry: config.officialNpmRegistry, + timeout: timeout || 60000 + }); + if (!r.data || !r.data.rows || !r.data.rows.length) { + return []; + } + + return r.data.rows.sort(function (a, b) { + return b.value - a.value; + }) + .slice(0, top) + .map(function (r) { + return r.key && r.key[0] + }) + .filter(function (r) { + return r; + }); +} diff --git a/proxy/sync_module_worker.js b/proxy/sync_module_worker.js index f410e80d0..fd8764f42 100644 --- a/proxy/sync_module_worker.js +++ b/proxy/sync_module_worker.js @@ -28,7 +28,6 @@ var crypto = require('crypto'); var sleep = require('co-sleep'); var urllib = require('../common/urllib'); var utility = require('utility'); -var ms = require('ms'); var urlparse = require('url').parse; var nfs = require('../common/nfs'); var npm = require('./npm'); diff --git a/sync/index.js b/sync/index.js index 8076bd2cd..6c4515f5f 100644 --- a/sync/index.js +++ b/sync/index.js @@ -16,7 +16,7 @@ var debug = require('debug')('cnpmjs.org:sync:index'); var co = require('co'); -var ms = require('ms'); +var ms = require('humanize-ms'); var util = require('util'); var utility = require('utility'); var config = require('../config'); @@ -58,7 +58,7 @@ var handleSync = co(function *() { var data; var error; try { - var data = yield *sync(); + data = yield *sync(); } catch (err) { error = err; error.message += ' (sync package error)'; @@ -77,6 +77,9 @@ if (sync) { setInterval(handleSync, ms(config.syncInterval)); } +/** + * sync dist(node.js and phantomjs) + */ var syncingDist = false; var syncDist = co(function* syncDist() { if (syncingDist) { @@ -104,6 +107,44 @@ if (config.syncDist) { logger.syncInfo('sync dist disable'); } +/** + * sync popular modules + */ + +var syncingPopular = false; +var syncPopular = co(function* syncPopular() { + if (syncingPopular) { + return; + } + syncingPopular = true; + logger.syncInfo('Start syncing popular modules...'); + var data; + var error; + try { + data = yield *require('./sync_popular'); + } catch (err) { + error = err; + error.message += ' (sync package error)'; + logger.syncError(error); + } + + if (data) { + logger.syncInfo(data); + } + if (!config.debug) { + sendMailToAdmin(error, data, new Date()); + } + + syncPopular = false; +}); + +if (config.syncPopular) { + syncPopular(); + setInterval(syncPopular, ms(config.syncPopularInterval)); +} else { + logger.syncInfo('sync popular module disable'); +} + function sendMailToAdmin(err, result, syncTime) { result = result || {}; var to = []; diff --git a/sync/sync_all.js b/sync/sync_all.js index 46c48c880..51b84f38d 100644 --- a/sync/sync_all.js +++ b/sync/sync_all.js @@ -15,7 +15,7 @@ */ var debug = require('debug')('cnpmjs.org:sync:sync_all'); -var ms = require('ms'); +var ms = require('humanize-ms'); var utility = require('utility'); var config = require('../config'); var Status = require('./status'); diff --git a/sync/sync_exist.js b/sync/sync_exist.js index 9844f1206..33c9ea2de 100644 --- a/sync/sync_exist.js +++ b/sync/sync_exist.js @@ -22,7 +22,7 @@ var SyncModuleWorker = require('../proxy/sync_module_worker'); var debug = require('debug')('cnpmjs.org:sync:sync_exist'); var utility = require('utility'); var Status = require('./status'); -var ms = require('ms'); +var ms = require('humanize-ms'); var thunkify = require('thunkify-wrap'); function intersection(arrOne, arrTwo) { diff --git a/sync/sync_popular.js b/sync/sync_popular.js new file mode 100644 index 000000000..979ace636 --- /dev/null +++ b/sync/sync_popular.js @@ -0,0 +1,47 @@ +/*! + * cnpmjs.org - sync/sync_popular.js + * + * Copyright(c) cnpmjs.org and other contributors. + * MIT Licensed + * + * Authors: + * dead_horse (http://deadhorse.me) + */ + +'use strict'; + +/** + * Module dependencies. + */ + +var SyncModuleWorker = require('../proxy/sync_module_worker'); +var debug = require('debug')('cnpmjs.org:sync:sync_popular'); +var thunkify = require('thunkify-wrap'); +var config = require('../config'); +var Npm = require('../proxy/npm'); +var utility = require('utility'); +var Status = require('./status'); + +module.exports = function *sync() { + var syncTime = Date.now(); + + var packages = yield Npm.getPopular(config.topPopular); + + var worker = new SyncModuleWorker({ + username: 'admin', + name: packages, + concurrency: config.syncConcurrency + }); + Status.init({need: packages.length}, worker); + worker.start(); + var end = thunkify.event(worker); + yield end(); + + debug('All popular packages sync done, successes %d, fails %d', + worker.successes.length, worker.fails.length); + + return { + successes: worker.successes, + fails: worker.fails + }; +}; diff --git a/test/proxy/npm.test.js b/test/proxy/npm.test.js index f22c0abdd..6a422bfd2 100644 --- a/test/proxy/npm.test.js +++ b/test/proxy/npm.test.js @@ -26,12 +26,12 @@ var fixtures = path.join(path.dirname(__dirname), 'fixtures'); describe('proxy/npm.test.js', function () { afterEach(mm.restore); - it('should return a module info from source npm', function *() { + it('should return a module info from source npm', function* () { var data = yield npm.get('pedding'); data.name.should.equal('pedding'); }); - it('should return null when module not exist', function *() { + it('should return null when module not exist', function* () { var data = yield npm.get('pedding-not-exists'); should.not.exist(data); }); @@ -46,7 +46,7 @@ describe('proxy/npm.test.js', function () { } }); - it('should return ServerError when http 500 response', function *() { + it('should return ServerError when http 500 response', function* () { var content = fs.createReadStream(path.join(fixtures, '500.txt')); mm.http.request(/\//, content, { statusCode: 500 }); // http://registry.npmjs.org/octopie @@ -58,4 +58,12 @@ describe('proxy/npm.test.js', function () { err.message.should.equal('Status 500, ' + fs.readFileSync(path.join(fixtures, '500.txt'), 'utf8')); } }); + + describe('getPopular()', function () { + it('should return popular modules', function* () { + var names = yield npm.getPopular(10); + names.should.have.a.lengthOf(10); + names[0].should.equal('underscore'); + }); + }); }); diff --git a/test/sync/sync_popular.test.js b/test/sync/sync_popular.test.js new file mode 100644 index 000000000..3c9cb1ef2 --- /dev/null +++ b/test/sync/sync_popular.test.js @@ -0,0 +1,32 @@ +/*! + * cnpmjs.org - test/sync/sync_popular.test.js + * + * Copyright(c) cnpmjs.org and other contributors. + * MIT Licensed + * + * Authors: + * dead_horse (http://deadhorse.me) + */ + +'use strict'; + +/** + * Module dependencies. + */ +var sync = require('../../sync/sync_popular'); +var mm = require('mm'); +var Npm = require('../../proxy/npm'); +var Total = require('../../proxy/total'); +var should = require('should'); + +describe('sync/sync_popular.test.js', function () { + describe('sync()', function () { + afterEach(mm.restore); + it('should sync popular modules ok', function *() { + mm.data(Npm, 'getPopular', ['mk2testmodule']); + var data = yield sync; + data.successes.should.eql(['mk2testmodule']); + mm.restore(); + }); + }); +});