-
Notifications
You must be signed in to change notification settings - Fork 6
/
cleanData.js
104 lines (73 loc) · 1.87 KB
/
cleanData.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
var configuration = require('./config/config.json');
var mongoose = require('mongoose');
var async = require('async');
var fs = require('fs');
console.log('Connecting to the database');
mongoose.connect(configuration.db);
var db = mongoose.connection;
db.once('open', function() {
var Tweet = require('./models/tweet.js');
/*Tweet
.find({
$where: 'this.date.toJSON().slice(0,10)=="2014-10-10"'
})
.exec(function(err, results) {
fs.writeFileSync('tweet20141010.json', JSON.stringify(results, '\n', true));
});*/
/*Tweet.aggregate({
$group: {
_id: {
year: {
$year: "$date"
},
month: {
$month: "$date"
},
day: {
$dayOfMonth: "$date"
},
},
count: {
$sum: 1
}
}
})
.exec(function(err, results) {
var count = {};
for (var i = 0; i < results.length; i++) {
var r = results[i];
var date = r._id.year + '-' + r._id.month + '-' + r._id.day;
console.log(date);
count[date] = r.count;
}
fs.writeFileSync('dayStats.json', JSON.stringify(count, '\t', true));
});*/
/*Tweet.aggregate({
$group: {
_id: '$userId',
count: {
$sum: 1
}
}
}, function(err, results) {
var count = {};
for (var i = 0; i < results.length; i++) {
var r = results[i];
var user = r._id;
count[user] = r.count;
}
fs.writeFileSync('userStats.json', JSON.stringify(count, '\t', true));
});*/
var tweets = [];
Tweet
.find({})
.exec(function(err, results) {
var replace = function(tweet, cb) {
tweet.text = tweet.text.replace(/\n/g, " ");
return tweet.save(cb);
};
async.each(results, replace, function() {
console.log('done');
});
});
});