add sync popular modules, close #468

config/index.js

@@ -207,6 +207,14 @@ var config = {
   // sync interval, default is 10 minutes
   syncInterval: '10m',
 
+  // sync polular modules, default to false
+  // because cnpm can't auto sync tag change for now
+  // so we want to sync popular modules to ensure their tags
+  syncPopular: false,
+  syncPopularInterval: '1h',
+  // top 100
+  topPopular: 100,
+
   // sync devDependencies or not, default is false
   syncDevDependencies: false,
 

controllers/registry/module.js

@@ -26,7 +26,6 @@ var coWrite = require('co-write');
 var urlparse = require('url').parse;
 var mime = require('mime');
 var semver = require('semver');
-var ms = require('ms');
 var config = require('../../config');
 var Module = require('../../proxy/module');
 var Total = require('../../proxy/total');

controllers/utils.js

@@ -18,7 +18,7 @@ var debug = require('debug')('cnpmjs.org:controllers:utils');
 var path = require('path');
 var fs = require('fs');
 var utility = require('utility');
-var ms = require('ms');
+var ms = require('humanize-ms');
 var nfs = require('../common/nfs');
 var config = require('../config');
 

package.json

@@ -30,6 +30,7 @@
     "giturl": "~0.0.3",
     "graceful": "~0.1.0",
     "gravatar": "~1.1.0",
+    "humanize-ms": "~1.0.0",
     "humanize-number": "~0.0.2",
     "koa": "~0.12.2",
     "koa-limit": "~1.0.2",
@@ -40,9 +41,8 @@
     "mini-logger": "~1.0.0",
     "mkdirp": "~0.5.0",
     "moment": "~2.8.3",
-    "ms": "~0.6.2",
     "multiline": "~1.0.1",
-    "mysql": "~2.5.1",
+    "mysql": "~2.5.2",
     "nodemailer": "~1.3.0",
     "ready": "~0.1.1",
     "redis": "~0.12.1",
@@ -59,7 +59,7 @@
     "istanbul-harmony": "~0.3.0",
     "jshint": "~2.5.6",
     "mm": "~0.2.1",
-    "mocha": "~1.21.4",
+    "mocha": "~1.21.5",
     "node-dev": "*",
     "pedding": "~1.0.0",
     "should": "~4.0.4",

proxy/npm.js

@@ -64,17 +64,38 @@ exports.get = function *(name) {
   return data;
 };
 
-exports.getAllSince = function *(startkey) {
+exports.getAllSince = function *(startkey, timeout) {
   var r = yield *request('/-/all/since?stale=update_after&startkey=' + startkey, {
-    timeout: 300000
+    timeout: timeout || 300000
   });
   return r.data;
 };
 
-exports.getShort = function *() {
+exports.getShort = function *(timeout) {
   var r = yield *request('/-/short', {
-    timeout: 300000,
+    timeout: timeout || 300000,
     registry: 'http://r.cnpmjs.org', // registry.npmjs.org/-/short is 404 now.
   });
   return r.data;
 };
+
+exports.getPopular = function* (top, timeout) {
+  var r = yield *request('/-/_view/dependedUpon?group_level=1', {
+    registry: config.officialNpmRegistry,
+    timeout: timeout || 60000
+  });
+  if (!r.data || !r.data.rows || !r.data.rows.length) {
+    return [];
+  }
+
+  return r.data.rows.sort(function (a, b) {
+    return b.value - a.value;
+  })
+  .slice(0, top)
+  .map(function (r) {
+    return r.key && r.key[0]
+  })
+  .filter(function (r) {
+    return r;
+  });
+}

proxy/sync_module_worker.js

@@ -28,7 +28,6 @@ var crypto = require('crypto');
 var sleep = require('co-sleep');
 var urllib = require('../common/urllib');
 var utility = require('utility');
-var ms = require('ms');
 var urlparse = require('url').parse;
 var nfs = require('../common/nfs');
 var npm = require('./npm');

sync/index.js

@@ -16,7 +16,7 @@
 
 var debug = require('debug')('cnpmjs.org:sync:index');
 var co = require('co');
-var ms = require('ms');
+var ms = require('humanize-ms');
 var util = require('util');
 var utility = require('utility');
 var config = require('../config');
@@ -58,7 +58,7 @@ var handleSync = co(function *() {
     var data;
     var error;
     try {
-      var data = yield *sync();
+      data = yield *sync();
     } catch (err) {
       error = err;
       error.message += ' (sync package error)';
@@ -77,6 +77,9 @@ if (sync) {
   setInterval(handleSync, ms(config.syncInterval));
 }
 
+/**
+ * sync dist(node.js and phantomjs)
+ */
 var syncingDist = false;
 var syncDist = co(function* syncDist() {
   if (syncingDist) {
@@ -104,6 +107,44 @@ if (config.syncDist) {
   logger.syncInfo('sync dist disable');
 }
 
+/**
+ * sync popular modules
+ */
+
+var syncingPopular = false;
+var syncPopular = co(function* syncPopular() {
+  if (syncingPopular) {
+    return;
+  }
+  syncingPopular = true;
+  logger.syncInfo('Start syncing popular modules...');
+  var data;
+  var error;
+  try {
+    data = yield *require('./sync_popular');
+  } catch (err) {
+    error = err;
+    error.message += ' (sync package error)';
+    logger.syncError(error);
+  }
+
+  if (data) {
+    logger.syncInfo(data);
+  }
+  if (!config.debug) {
+    sendMailToAdmin(error, data, new Date());
+  }
+
+  syncPopular = false;
+});
+
+if (config.syncPopular) {
+  syncPopular();
+  setInterval(syncPopular, ms(config.syncPopularInterval));
+} else {
+  logger.syncInfo('sync popular module disable');
+}
+
 function sendMailToAdmin(err, result, syncTime) {
   result = result || {};
   var to = [];

sync/sync_all.js

@@ -15,7 +15,7 @@
  */
 
 var debug = require('debug')('cnpmjs.org:sync:sync_all');
-var ms = require('ms');
+var ms = require('humanize-ms');
 var utility = require('utility');
 var config = require('../config');
 var Status = require('./status');

sync/sync_exist.js

@@ -22,7 +22,7 @@ var SyncModuleWorker = require('../proxy/sync_module_worker');
 var debug = require('debug')('cnpmjs.org:sync:sync_exist');
 var utility = require('utility');
 var Status = require('./status');
-var ms = require('ms');
+var ms = require('humanize-ms');
 var thunkify = require('thunkify-wrap');
 
 function intersection(arrOne, arrTwo) {

sync/sync_popular.js

@@ -0,0 +1,47 @@
+/*!
+ * cnpmjs.org - sync/sync_popular.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <[email protected]> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+
+var SyncModuleWorker = require('../proxy/sync_module_worker');
+var debug = require('debug')('cnpmjs.org:sync:sync_popular');
+var thunkify = require('thunkify-wrap');
+var config = require('../config');
+var Npm = require('../proxy/npm');
+var utility = require('utility');
+var Status = require('./status');
+
+module.exports = function *sync() {
+  var syncTime = Date.now();
+
+  var packages = yield Npm.getPopular(config.topPopular);
+
+  var worker = new SyncModuleWorker({
+    username: 'admin',
+    name: packages,
+    concurrency: config.syncConcurrency
+  });
+  Status.init({need: packages.length}, worker);
+  worker.start();
+  var end = thunkify.event(worker);
+  yield end();
+
+  debug('All popular packages sync done, successes %d, fails %d',
+    worker.successes.length, worker.fails.length);
+
+  return {
+    successes: worker.successes,
+    fails: worker.fails
+  };
+};

test/proxy/npm.test.js

@@ -26,12 +26,12 @@ var fixtures = path.join(path.dirname(__dirname), 'fixtures');
 describe('proxy/npm.test.js', function () {
   afterEach(mm.restore);
 
-  it('should return a module info from source npm', function *() {
+  it('should return a module info from source npm', function* () {
     var data = yield npm.get('pedding');
     data.name.should.equal('pedding');
   });
 
-  it('should return null when module not exist', function *() {
+  it('should return null when module not exist', function* () {
     var data = yield npm.get('pedding-not-exists');
     should.not.exist(data);
   });
@@ -46,7 +46,7 @@ describe('proxy/npm.test.js', function () {
     }
   });
 
-  it('should return ServerError when http 500 response', function *() {
+  it('should return ServerError when http 500 response', function* () {
     var content = fs.createReadStream(path.join(fixtures, '500.txt'));
     mm.http.request(/\//, content, { statusCode: 500 });
     // http://registry.npmjs.org/octopie
@@ -58,4 +58,12 @@ describe('proxy/npm.test.js', function () {
       err.message.should.equal('Status 500, ' + fs.readFileSync(path.join(fixtures, '500.txt'), 'utf8'));
     }
   });
+
+  describe('getPopular()', function () {
+    it('should return popular modules', function* () {
+      var names = yield npm.getPopular(10);
+      names.should.have.a.lengthOf(10);
+      names[0].should.equal('underscore');
+    });
+  });
 });

test/sync/sync_popular.test.js

@@ -0,0 +1,32 @@
+/*!
+ * cnpmjs.org - test/sync/sync_popular.test.js
+ *
+ * Copyright(c) cnpmjs.org and other contributors.
+ * MIT Licensed
+ *
+ * Authors:
+ *  dead_horse <[email protected]> (http://deadhorse.me)
+ */
+
+'use strict';
+
+/**
+ * Module dependencies.
+ */
+var sync = require('../../sync/sync_popular');
+var mm = require('mm');
+var Npm = require('../../proxy/npm');
+var Total = require('../../proxy/total');
+var should = require('should');
+
+describe('sync/sync_popular.test.js', function () {
+  describe('sync()', function () {
+    afterEach(mm.restore);
+    it('should sync popular modules ok', function *() {
+      mm.data(Npm, 'getPopular', ['mk2testmodule']);
+      var data = yield sync;
+      data.successes.should.eql(['mk2testmodule']);
+      mm.restore();
+    });
+  });
+});