From 4f195ddbf6615314a3bd51d115d30ab87a00bbcf Mon Sep 17 00:00:00 2001 From: Ben West Date: Sat, 2 Aug 2014 00:26:52 -0700 Subject: [PATCH 001/205] introduce mqtt Introduce basic mqtt. --- env.js | 1 + lib/mqtt.js | 38 ++++++++++++++++++++++++++++++++++++++ package.json | 3 ++- server.js | 5 +++++ 4 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 lib/mqtt.js diff --git a/env.js b/env.js index 1ec8ab1d334..8a1e3f2d82f 100644 --- a/env.js +++ b/env.js @@ -21,6 +21,7 @@ function config ( ) { env.version = software.version; env.name = software.name; + env.MQTT_MONITOR = process.env.MQTT_MONITOR || null; env.DISPLAY_UNITS = process.env.DISPLAY_UNITS || 'mg/dl'; env.PORT = process.env.PORT || 1337; env.mongo = process.env.MONGO_CONNECTION || process.env.CUSTOMCONNSTR_mongo; diff --git a/lib/mqtt.js b/lib/mqtt.js new file mode 100644 index 00000000000..5b129b705f6 --- /dev/null +++ b/lib/mqtt.js @@ -0,0 +1,38 @@ +'use strict'; + +var es = require('event-stream'); +var mqtt = require('mqtt'); + +function process (client) { + var stream = es.through( + function _write (data) { + this.push(data); + } + ); + return stream; +} + +function every (storage) { + function iter (item, next) { + storage.create(item, next); + } + return es.map(iter); +} + +function configure (env) { + var uri = env['MQTT_MONITOR']; + var client = mqtt.connect(uri); + client.subscribe('sgvs'); + client.subscribe('published'); + client.subscribe('entries/sgv', function ( ) { + console.log('granted', arguments); + }); + + client.on('message', function (topic, msg) { console.log('topic', topic); + console.log('msg', msg); + }); + client.entries = process(client); + client.every = every; + return client; +} +module.exports = configure; diff --git a/package.json b/package.json index c0f768dee75..9df7a621a90 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,8 @@ "express-extension-to-accept": "0.0.2", "mongodb": "^1.4.7", "sgvdata": "0.0.2", - "socket.io": "^0.9.17" + "socket.io": "^0.9.17", + "mqtt": "~0.3.11" }, "devDependencies": { "supertest": "~0.13.0", diff --git a/server.js b/server.js index 8d8e460fece..a2dbfadf95c 100644 --- a/server.js +++ b/server.js @@ -72,6 +72,11 @@ store(function ready ( ) { var server = app.listen(PORT); console.log('listening', PORT); + if (env.MQTT_MONITOR) { + var mqtt = require('./lib/mqtt')(env); + mqtt.entries.pipe(mqtt.every(entries)); + } + /////////////////////////////////////////////////// // setup socket io for data and message transmission /////////////////////////////////////////////////// From cc0cd36934faf5984d6b4eb3faa1dff575d95616 Mon Sep 17 00:00:00 2001 From: Ben West Date: Sat, 2 Aug 2014 00:32:51 -0700 Subject: [PATCH 002/205] lint records first --- server.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server.js b/server.js index a2dbfadf95c..93c47b3e0ec 100644 --- a/server.js +++ b/server.js @@ -74,7 +74,8 @@ store(function ready ( ) { if (env.MQTT_MONITOR) { var mqtt = require('./lib/mqtt')(env); - mqtt.entries.pipe(mqtt.every(entries)); + var es = require('event-stream'); + es.pipeline(mqtt.entries, entries.map( ), mqtt.every(entries)); } /////////////////////////////////////////////////// From b9e8a5faf11f21fc752dc201df4c048dbf8c1cfc Mon Sep 17 00:00:00 2001 From: Ben West Date: Wed, 1 Oct 2014 13:50:31 -0700 Subject: [PATCH 003/205] make mqtt experiments suitable for hacking Uses experimental support in sgvdata for protobuf. --- lib/mqtt.js | 16 ++++++++++++++++ package.json | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/mqtt.js b/lib/mqtt.js index 5b129b705f6..eddd4d6a8e5 100644 --- a/lib/mqtt.js +++ b/lib/mqtt.js @@ -1,6 +1,7 @@ 'use strict'; var es = require('event-stream'); +var decoders = require('sgvdata/lib/protobuf'); var mqtt = require('mqtt'); function process (client) { @@ -18,16 +19,31 @@ function every (storage) { } return es.map(iter); } +function downloader ( ) { + var opts = { + model: decoders.models.CookieMonsterG4Download + , json: function (o) { return o; } + , payload: function (o) { return o; } + }; + return decoders(opts); +} function configure (env) { var uri = env['MQTT_MONITOR']; var client = mqtt.connect(uri); + var downloads = downloader( ); client.subscribe('sgvs'); client.subscribe('published'); + client.subscribe('/downloads/protobuf'); client.subscribe('entries/sgv', function ( ) { console.log('granted', arguments); }); + client.on('/downloads/protobuf', function (topic, msg) { console.log('topic', topic); + + console.log('DOWNLOAD msg', msg, downloads.parse(msg)); + }); + client.on('message', function (topic, msg) { console.log('topic', topic); console.log('msg', msg); }); diff --git a/package.json b/package.json index b835ffaf7ac..e60fa52612b 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,7 @@ "mongodb": "^1.4.7", "moment": "2.8.1", "pushover-notifications": "0.2.0", - "sgvdata": "0.0.2", + "sgvdata": "git://github.com/bewest/sgvdata.git#wip/protobuf", "socket.io": "^0.9.17", "mqtt": "~0.3.11", "git-rev": "git://github.com/bewest/git-rev.git" From a22ec4c5349aeba9299bf876bcf69cb29aeb2392 Mon Sep 17 00:00:00 2001 From: Ben West Date: Wed, 19 Nov 2014 13:46:33 -0800 Subject: [PATCH 004/205] add forever.js: make server run forever This helps with heroku and other non-azure hosts. --- Procfile | 2 +- package.json | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Procfile b/Procfile index 489b2700aca..b62d80ef5af 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: node server.js +web: ./node_modules/.bin/forever server.js diff --git a/package.json b/package.json index 9fa88c37110..1b2f4675ab8 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,8 @@ "pushover-notifications": "0.2.0", "sgvdata": "0.0.2", "socket.io": "^0.9.17", - "git-rev": "git://github.com/bewest/git-rev.git" + "git-rev": "git://github.com/bewest/git-rev.git", + "forever": "~0.13.0" }, "devDependencies": { "supertest": "~0.13.0", From 079fe495db93a6f4dc1b436ff63ca623366d01f7 Mon Sep 17 00:00:00 2001 From: Ben West Date: Fri, 21 Nov 2014 12:06:29 -0800 Subject: [PATCH 005/205] tweak how forever runs --- Procfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Procfile b/Procfile index b62d80ef5af..f5d45241167 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: ./node_modules/.bin/forever server.js +web: ./node_modules/.bin/forever start -c server.js From 812d028008912f361b8df80b02832d754833f05a Mon Sep 17 00:00:00 2001 From: Ben West Date: Fri, 21 Nov 2014 15:24:59 -0800 Subject: [PATCH 006/205] tweak forever to restart things Found that the parameters didn't work as advertised unless done like this. --- Procfile | 2 +- lib/storage.js | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Procfile b/Procfile index f5d45241167..32dd1c83adc 100644 --- a/Procfile +++ b/Procfile @@ -1 +1 @@ -web: ./node_modules/.bin/forever start -c server.js +web: ./node_modules/.bin/forever --minUptime 100 -c node server.js diff --git a/lib/storage.js b/lib/storage.js index 58a965a7d87..6a02a83f25a 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -1,5 +1,7 @@ 'use strict'; var mongodb = require('mongodb'); +var levelup = require('levelup'); +var mongodown = require('mongo-down'); function init (env, cb) { var MongoClient = mongodb.MongoClient; @@ -11,6 +13,9 @@ function init (env, cb) { if (cb && cb.call) { cb(null, mongo); } return; } + if (!env.mongo) { + throw new Error("Mongo string is missing"); + } console.log("Connecting to mongo"); MongoClient.connect(env.mongo, function connected (err, db) { if (err) { From 4f0758a8a83c2f0ec4d2448bad5c048daaba1c0a Mon Sep 17 00:00:00 2001 From: Ben West Date: Fri, 21 Nov 2014 15:34:07 -0800 Subject: [PATCH 007/205] oops, remove spurious modules --- lib/storage.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/storage.js b/lib/storage.js index 6a02a83f25a..d5b128a0592 100644 --- a/lib/storage.js +++ b/lib/storage.js @@ -1,7 +1,5 @@ 'use strict'; var mongodb = require('mongodb'); -var levelup = require('levelup'); -var mongodown = require('mongo-down'); function init (env, cb) { var MongoClient = mongodb.MongoClient; From c8be1deb9d49c7efa3c20a1418b1a58df42921b6 Mon Sep 17 00:00:00 2001 From: Ben West Date: Sat, 22 Nov 2014 16:40:48 -0800 Subject: [PATCH 008/205] get mqtt basically working, very rough --- lib/entries.js | 5 ++++ lib/mqtt.js | 79 +++++++++++++++++++++++++++++++++++++++++++------- server.js | 2 +- 3 files changed, 75 insertions(+), 11 deletions(-) diff --git a/lib/entries.js b/lib/entries.js index 1746bcefa5e..19f023a9278 100644 --- a/lib/entries.js +++ b/lib/entries.js @@ -140,6 +140,11 @@ function entries (name, storage) { }); } + function writeStream (opts) { + function map (item, next) { + } + } + // closure to represent the API function api ( ) { // obtain handle usable for querying the collection associated diff --git a/lib/mqtt.js b/lib/mqtt.js index eddd4d6a8e5..e08f5069ca4 100644 --- a/lib/mqtt.js +++ b/lib/mqtt.js @@ -1,7 +1,9 @@ 'use strict'; var es = require('event-stream'); +var Long = require('long'); var decoders = require('sgvdata/lib/protobuf'); +var direction = require('sgvdata/lib/utils').direction; var mqtt = require('mqtt'); function process (client) { @@ -28,24 +30,81 @@ function downloader ( ) { return decoders(opts); } -function configure (env) { +function toSGV (proto) { + var ts = long_time(proto.timestamp); + var obj = { + device: 'dexcom' + , date: ts.getTime( ) + , dateString: ts.toISOString( ) + , sgv: proto.sgv + , direction: direction(proto.direction) + , type: 'sgv' + }; + return obj; +} + +function createProtoStream (packet) { + var stream = es.readArray(packet.sgv); + function map (item, next) { + var r = toSGV(item); + console.log("ITEM", item, "TO SGV", r); + next(null, r); + } + return stream.pipe(es.map(map)); +} +function long_time (p) { + var ts = parseInt(new Long(p.low, p.high, p.unsigned).toString( )); + return new Date(ts); +} + +function configure (env, core) { var uri = env['MQTT_MONITOR']; - var client = mqtt.connect(uri); + var opts = {encoding: 'binary'}; + var client = mqtt.connect(uri, opts); var downloads = downloader( ); client.subscribe('sgvs'); client.subscribe('published'); - client.subscribe('/downloads/protobuf'); - client.subscribe('entries/sgv', function ( ) { + client.subscribe('/downloads/protobuf', granted); + client.subscribe('/uploader', granted); + client.subscribe('/entries/sgv', granted); + function granted ( ) { console.log('granted', arguments); - }); - - client.on('/downloads/protobuf', function (topic, msg) { console.log('topic', topic); + } - console.log('DOWNLOAD msg', msg, downloads.parse(msg)); - }); client.on('message', function (topic, msg) { console.log('topic', topic); - console.log('msg', msg); + console.log(topic, 'on message', 'msg', msg.length); + switch (topic) { + case '/uploader': + console.log({type: topic, msg: msg.toString( )}); + break; + case '/downloads/protobuf': + var b = new Buffer(msg, 'binary'); + console.log("BINARY", b.length, b.toString('hex')); + try { + var packet = downloads.parse(b); + if (!packet.type) { + packet.type = topic; + } + } catch (e) { + console.log("DID NOT PARSE", e); + break; + } + console.log('DOWNLOAD msg', msg.length, packet); + console.log('download SGV', packet.sgv[0]); + console.log('download_timestamp', packet.download_timestamp, long_time(packet.download_timestamp)); + console.log("WRITE TO MONGO"); + createProtoStream(packet).pipe(core.persist(function empty(err, result) { + console.log("DONE WRITING TO MONGO", err); + })); + + // core.write(packet); + break; + default: + console.log(topic, 'on message', 'msg', msg); + // core.write(msg); + break; + } }); client.entries = process(client); client.every = every; diff --git a/server.js b/server.js index 2bc3e4a3617..374cde658fa 100644 --- a/server.js +++ b/server.js @@ -83,7 +83,7 @@ store(function ready ( ) { console.log('listening', PORT); if (env.MQTT_MONITOR) { - var mqtt = require('./lib/mqtt')(env); + var mqtt = require('./lib/mqtt')(env, entries); var es = require('event-stream'); es.pipeline(mqtt.entries, entries.map( ), mqtt.every(entries)); } From 756e9e81d440d1bb241f6bdd7eebad10267a8332 Mon Sep 17 00:00:00 2001 From: Ben West Date: Sat, 22 Nov 2014 17:55:36 -0800 Subject: [PATCH 009/205] forgot long --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index e60fa52612b..0ecff09ee10 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,8 @@ "sgvdata": "git://github.com/bewest/sgvdata.git#wip/protobuf", "socket.io": "^0.9.17", "mqtt": "~0.3.11", - "git-rev": "git://github.com/bewest/git-rev.git" + "git-rev": "git://github.com/bewest/git-rev.git", + "long": "~2.2.3" }, "devDependencies": { "supertest": "~0.13.0", From 9ad7bf1b2882095fcaeff80b91239fdf4b813d84 Mon Sep 17 00:00:00 2001 From: Ben West Date: Tue, 25 Nov 2014 12:26:09 -0800 Subject: [PATCH 010/205] test coverage --- Makefile | 3 +++ package.json | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/Makefile b/Makefile index bb605d08d82..521d5672779 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,9 @@ coveralls: ${TESTS} | ./coverall.sh coverhtml: + MONGO_CONNECTION=${MONGO_CONNECTION} \ + CUSTOMCONNSTR_mongo_collection=${CUSTOMCONNSTR_mongo_collection} \ + CUSTOMCONNSTR_mongo_settings_collection=${CUSTOMCONNSTR_mongo_settings_collection} \ ./node_modules/.bin/mocha ${BLANKET} -R html-cov ${TESTS} > tests/coverage.html test: diff --git a/package.json b/package.json index 0ecff09ee10..2832f403c80 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,16 @@ "test": "make test", "postinstall": "node node_modules/bower/bin/bower install" }, + "config": { + "blanket": { + "pattern": [ + "tests", "lib", "server", "app", "static/js" + ], + "data-cover-never": [ + "node_modules" + ] + } + }, "engines": { "node": ">=0.10 <0.12" }, From 33e1a36726cd60ed5c3b8aee77c9534a2458732f Mon Sep 17 00:00:00 2001 From: Kevin Lee Date: Fri, 26 Dec 2014 03:57:10 -0600 Subject: [PATCH 011/205] Quick hack to get things working --- lib/mqtt.js | 20 +++++++++++++------- package.json | 2 +- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/lib/mqtt.js b/lib/mqtt.js index e08f5069ca4..fe56ed379f8 100644 --- a/lib/mqtt.js +++ b/lib/mqtt.js @@ -5,6 +5,7 @@ var Long = require('long'); var decoders = require('sgvdata/lib/protobuf'); var direction = require('sgvdata/lib/utils').direction; var mqtt = require('mqtt'); +var moment = require('moment'); function process (client) { var stream = es.through( @@ -31,13 +32,15 @@ function downloader ( ) { } function toSGV (proto) { - var ts = long_time(proto.timestamp); + var ts = moment(proto.download_timestamp); + console.log("TIMESTAMP", moment(proto.download_timestamp)); var obj = { device: 'dexcom' - , date: ts.getTime( ) - , dateString: ts.toISOString( ) - , sgv: proto.sgv - , direction: direction(proto.direction) + , date: ts.unix() * 1000 + , dateString: ts.format() + , sgv: proto.sgv_mgdl + , direction: direction(proto.trend) + ,noise: proto.noise , type: 'sgv' }; return obj; @@ -59,12 +62,15 @@ function long_time (p) { function configure (env, core) { var uri = env['MQTT_MONITOR']; - var opts = {encoding: 'binary'}; + var opts = { + encoding: 'binary', + clean: false + }; var client = mqtt.connect(uri, opts); var downloads = downloader( ); client.subscribe('sgvs'); client.subscribe('published'); - client.subscribe('/downloads/protobuf', granted); + client.subscribe('/downloads/protobuf',{qos: 2}, granted); client.subscribe('/uploader', granted); client.subscribe('/entries/sgv', granted); function granted ( ) { diff --git a/package.json b/package.json index 2832f403c80..c040d8b4f74 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "mongodb": "^1.4.7", "moment": "2.8.1", "pushover-notifications": "0.2.0", - "sgvdata": "git://github.com/bewest/sgvdata.git#wip/protobuf", + "sgvdata": "git://github.com/ktind/sgvdata.git#wip/protobuf", "socket.io": "^0.9.17", "mqtt": "~0.3.11", "git-rev": "git://github.com/bewest/git-rev.git", From 1f1bc4d6f95907025cc81944c9d1dcc8e35bdab3 Mon Sep 17 00:00:00 2001 From: Ben West Date: Sat, 27 Dec 2014 11:14:41 -0800 Subject: [PATCH 012/205] blarg, attempt to debug mqtt --- lib/mqtt.js | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/mqtt.js b/lib/mqtt.js index fe56ed379f8..efcba12479c 100644 --- a/lib/mqtt.js +++ b/lib/mqtt.js @@ -31,16 +31,21 @@ function downloader ( ) { return decoders(opts); } +function ReceiverTime (ts) { + var base = Date.parse('2009-01-01T00:00:00-0800'); + return new Date(base + (ts * 1000)); +} + function toSGV (proto) { var ts = moment(proto.download_timestamp); - console.log("TIMESTAMP", moment(proto.download_timestamp)); + console.log("errr", proto, "TIMESTAMP", ReceiverTime(proto.timestamp_sec)); var obj = { device: 'dexcom' , date: ts.unix() * 1000 , dateString: ts.format() , sgv: proto.sgv_mgdl , direction: direction(proto.trend) - ,noise: proto.noise + , noise: proto.noise , type: 'sgv' }; return obj; @@ -64,7 +69,8 @@ function configure (env, core) { var uri = env['MQTT_MONITOR']; var opts = { encoding: 'binary', - clean: false + clean: false, + clientId: 'master' }; var client = mqtt.connect(uri, opts); var downloads = downloader( ); @@ -98,7 +104,7 @@ function configure (env, core) { } console.log('DOWNLOAD msg', msg.length, packet); console.log('download SGV', packet.sgv[0]); - console.log('download_timestamp', packet.download_timestamp, long_time(packet.download_timestamp)); + console.log('download_timestamp', packet.download_timestamp, Date.parse(packet.download_timestamp)); console.log("WRITE TO MONGO"); createProtoStream(packet).pipe(core.persist(function empty(err, result) { console.log("DONE WRITING TO MONGO", err); From f70a031357f3d10b27ee91ae855018710ea0d4be Mon Sep 17 00:00:00 2001 From: Ben West Date: Tue, 30 Dec 2014 11:56:32 -0800 Subject: [PATCH 013/205] store each record according to own timestamp --- lib/mqtt.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mqtt.js b/lib/mqtt.js index efcba12479c..25e1159ca51 100644 --- a/lib/mqtt.js +++ b/lib/mqtt.js @@ -37,7 +37,7 @@ function ReceiverTime (ts) { } function toSGV (proto) { - var ts = moment(proto.download_timestamp); + var ts = moment(ReceiverTime(proto.timestamp_sec)); console.log("errr", proto, "TIMESTAMP", ReceiverTime(proto.timestamp_sec)); var obj = { device: 'dexcom' From bf2f6231da6635050395f80154256e53f5327557 Mon Sep 17 00:00:00 2001 From: Ben West Date: Tue, 30 Dec 2014 19:06:40 -0800 Subject: [PATCH 014/205] tweak times and protobuf model with @ktind --- lib/mqtt.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/mqtt.js b/lib/mqtt.js index 25e1159ca51..38d26525fd9 100644 --- a/lib/mqtt.js +++ b/lib/mqtt.js @@ -24,7 +24,7 @@ function every (storage) { } function downloader ( ) { var opts = { - model: decoders.models.CookieMonsterG4Download + model: decoders.models.CookieMonsterDownload , json: function (o) { return o; } , payload: function (o) { return o; } }; @@ -37,8 +37,8 @@ function ReceiverTime (ts) { } function toSGV (proto) { - var ts = moment(ReceiverTime(proto.timestamp_sec)); - console.log("errr", proto, "TIMESTAMP", ReceiverTime(proto.timestamp_sec)); + var ts = moment(ReceiverTime(proto.disp_timestamp_sec)); + console.log("errr", proto, "TIMESTAMP", ReceiverTime(proto.disp_timestamp_sec)); var obj = { device: 'dexcom' , date: ts.unix() * 1000 From 70877f7f67295e6fabfddc9af93ac9986e34d335 Mon Sep 17 00:00:00 2001 From: Kevin Lee Date: Tue, 30 Dec 2014 21:08:34 -0600 Subject: [PATCH 015/205] Adding experimental time calculation algorithm --- lib/mqtt.js | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/lib/mqtt.js b/lib/mqtt.js index 25e1159ca51..1d088ceffc7 100644 --- a/lib/mqtt.js +++ b/lib/mqtt.js @@ -24,7 +24,7 @@ function every (storage) { } function downloader ( ) { var opts = { - model: decoders.models.CookieMonsterG4Download + model: decoders.models.CookieMonsterDownload , json: function (o) { return o; } , payload: function (o) { return o; } }; @@ -36,13 +36,21 @@ function ReceiverTime (ts) { return new Date(base + (ts * 1000)); } -function toSGV (proto) { - var ts = moment(ReceiverTime(proto.timestamp_sec)); - console.log("errr", proto, "TIMESTAMP", ReceiverTime(proto.timestamp_sec)); +function toSGV (proto, receiver_time, download_time) { + var ts = moment(download_time); + console.log("Receiver time: ", receiver_time); + console.log("Record time: ", proto.sys_timestamp_sec); + console.log("Download time: ", ts.unix()); + var record_offset = receiver_time - proto.sys_timestamp_sec; + var record_time = ts.subtract(record_offset, 'second'); + + console.log("errr", " Offset: ",record_offset, " Record time: ", record_time.format()); + + //console.log("errr", proto, "TIMESTAMP", ReceiverTime(proto.disp_timestamp_sec)); var obj = { device: 'dexcom' - , date: ts.unix() * 1000 - , dateString: ts.format() + , date: record_time.unix() * 1000 + , dateString: record_time.format() , sgv: proto.sgv_mgdl , direction: direction(proto.trend) , noise: proto.noise @@ -53,8 +61,10 @@ function toSGV (proto) { function createProtoStream (packet) { var stream = es.readArray(packet.sgv); + var receiver_time = packet.receiver_system_time_sec; + var download_time = packet.download_timestamp; function map (item, next) { - var r = toSGV(item); + var r = toSGV(item, receiver_time, download_time); console.log("ITEM", item, "TO SGV", r); next(null, r); } From 650bcd1aae32fd3611979614c0a8dbd90d94b270 Mon Sep 17 00:00:00 2001 From: Kevin Lee Date: Wed, 31 Dec 2014 03:02:46 -0600 Subject: [PATCH 016/205] First pass at adding MGB, Sensor, Calibration, and Device Status records via MQTT New method to create device status with a backdated timestamp --- lib/devicestatus.js | 69 +++++---- lib/mqtt.js | 336 +++++++++++++++++++++++++++++--------------- server.js | 2 +- 3 files changed, 265 insertions(+), 142 deletions(-) diff --git a/lib/devicestatus.js b/lib/devicestatus.js index fb21d1c2490..9b9cf881364 100644 --- a/lib/devicestatus.js +++ b/lib/devicestatus.js @@ -1,34 +1,43 @@ 'use strict'; -function configure (collection, storage) { - - function create (obj, fn) { - obj.created_at = (new Date( )).toISOString( ); - api( ).insert(obj, function (err, doc) { - fn(null, doc); - }); - } - - function last(fn) { - return api( ).find({ }).sort({created_at: -1}).limit(1).toArray(function(err, entries) { - if (entries && entries.length > 0) - fn(err, entries[0]); - else - fn(err, null); - }); - } - - function list (fn) { - return api( ).find({ }).sort({created_at: -1}).toArray(fn); - } - - function api ( ) { - return storage.pool.db.collection(collection); - } - - api.list = list; - api.create = create; - api.last = last; - return api; +function configure(collection, storage) { + + function create(obj, fn) { + if (! obj.hasOwnProperty("created_at")){ + obj.created_at = (new Date()).toISOString(); + } + api().insert(obj, function (err, doc) { + fn(null, doc); + }); + } + + function create_date_included(obj, fn) { + api().insert(obj, function (err, doc) { + fn(null, doc); + }); + + } + + function last(fn) { + return api().find({}).sort({created_at: -1}).limit(1).toArray(function (err, entries) { + if (entries && entries.length > 0) + fn(err, entries[0]); + else + fn(err, null); + }); + } + + function list(fn) { + return api().find({}).sort({created_at: -1}).toArray(fn); + } + + function api() { + return storage.pool.db.collection(collection); + } + + api.list = list; + api.create = create; + api.last = last; + return api; } module.exports = configure; diff --git a/lib/mqtt.js b/lib/mqtt.js index 1d088ceffc7..6da3253415b 100644 --- a/lib/mqtt.js +++ b/lib/mqtt.js @@ -1,135 +1,249 @@ 'use strict'; var es = require('event-stream'); -var Long = require('long'); var decoders = require('sgvdata/lib/protobuf'); var direction = require('sgvdata/lib/utils').direction; var mqtt = require('mqtt'); var moment = require('moment'); -function process (client) { - var stream = es.through( - function _write (data) { - this.push(data); +function process(client) { + var stream = es.through( + function _write(data) { + this.push(data); + } + ); + return stream; +} + +function every(storage) { + function iter(item, next) { + storage.create(item, next); } - ); - return stream; + + return es.map(iter); } -function every (storage) { - function iter (item, next) { - storage.create(item, next); - } - return es.map(iter); +function downloader() { + var opts = { + model: decoders.models.CookieMonsterDownload + , json: function (o) { + return o; + } + , payload: function (o) { + return o; + } + }; + return decoders(opts); } -function downloader ( ) { - var opts = { - model: decoders.models.CookieMonsterDownload - , json: function (o) { return o; } - , payload: function (o) { return o; } - }; - return decoders(opts); + +function ReceiverTime(ts) { + var base = Date.parse('2009-01-01T00:00:00-0800'); + return new Date(base + (ts * 1000)); } -function ReceiverTime (ts) { - var base = Date.parse('2009-01-01T00:00:00-0800'); - return new Date(base + (ts * 1000)); +function toSGV(proto, receiver_time, download_time) { + console.log("Receiver time: ", receiver_time); + console.log("Record time: ", proto.sys_timestamp_sec); + console.log("Download time: ", download_time.unix()); + var record_offset = receiver_time - proto.sys_timestamp_sec; + var record_time = download_time.subtract(record_offset, 'second'); + + console.log("errr", " Offset: ", record_offset, " Record time: ", record_time.format()); + + var obj = { + device: 'dexcom' + , date: record_time.unix() * 1000 + , dateString: record_time.format() + , sgv: proto.sgv_mgdl + , direction: direction(proto.trend) + , noise: proto.noise + , type: 'sgv' + }; + return obj; } -function toSGV (proto, receiver_time, download_time) { - var ts = moment(download_time); - console.log("Receiver time: ", receiver_time); - console.log("Record time: ", proto.sys_timestamp_sec); - console.log("Download time: ", ts.unix()); - var record_offset = receiver_time - proto.sys_timestamp_sec; - var record_time = ts.subtract(record_offset, 'second'); - - console.log("errr", " Offset: ",record_offset, " Record time: ", record_time.format()); - - //console.log("errr", proto, "TIMESTAMP", ReceiverTime(proto.disp_timestamp_sec)); - var obj = { - device: 'dexcom' - , date: record_time.unix() * 1000 - , dateString: record_time.format() - , sgv: proto.sgv_mgdl - , direction: direction(proto.trend) - , noise: proto.noise - , type: 'sgv' - }; - return obj; +function createProtoStream(packet, download_time) { + var stream = es.readArray(packet.sgv); + var receiver_time = packet.receiver_system_time_sec; + + function map(item, next) { + var r = toSGV(item, receiver_time, download_time); + console.log("ITEM", item, "TO SGV", r); + next(null, r); + } + + return stream.pipe(es.map(map)); } -function createProtoStream (packet) { - var stream = es.readArray(packet.sgv); - var receiver_time = packet.receiver_system_time_sec; - var download_time = packet.download_timestamp; - function map (item, next) { - var r = toSGV(item, receiver_time, download_time); - console.log("ITEM", item, "TO SGV", r); - next(null, r); - } - return stream.pipe(es.map(map)); +function toCal(proto, receiver_time, download_time) { + console.log("Receiver time: ", receiver_time); + console.log("Record time: ", proto.sys_timestamp_sec); + console.log("Download time: ", download_time.unix()); + var record_offset = receiver_time - proto.sys_timestamp_sec; + var record_time = download_time.subtract(record_offset, 'second'); + + console.log("errr", " Offset: ", record_offset, " Record time: ", record_time.format()); + + var obj = { + device: 'dexcom' + , date: record_time.unix() * 1000 + , dateString: record_time.format() + , slope: proto.slope + , intercept: proto.intercept + , scale: proto.scale + , type: 'cal' + }; + return obj; } -function long_time (p) { - var ts = parseInt(new Long(p.low, p.high, p.unsigned).toString( )); - return new Date(ts); + +function createCalProtoStream(packet, download_time) { + var stream = es.readArray(packet.cal); + var receiver_time = packet.receiver_system_time_sec; + + function map(item, next) { + var r = toCal(item, receiver_time, download_time); + console.log("ITEM", item, "TO CAL", r); + next(null, r); + } + + return stream.pipe(es.map(map)); } -function configure (env, core) { - var uri = env['MQTT_MONITOR']; - var opts = { - encoding: 'binary', - clean: false, - clientId: 'master' - }; - var client = mqtt.connect(uri, opts); - var downloads = downloader( ); - client.subscribe('sgvs'); - client.subscribe('published'); - client.subscribe('/downloads/protobuf',{qos: 2}, granted); - client.subscribe('/uploader', granted); - client.subscribe('/entries/sgv', granted); - function granted ( ) { - console.log('granted', arguments); - } - - - client.on('message', function (topic, msg) { console.log('topic', topic); - console.log(topic, 'on message', 'msg', msg.length); - switch (topic) { - case '/uploader': - console.log({type: topic, msg: msg.toString( )}); - break; - case '/downloads/protobuf': - var b = new Buffer(msg, 'binary'); - console.log("BINARY", b.length, b.toString('hex')); - try { - var packet = downloads.parse(b); - if (!packet.type) { - packet.type = topic; - } - } catch (e) { - console.log("DID NOT PARSE", e); - break; - } - console.log('DOWNLOAD msg', msg.length, packet); - console.log('download SGV', packet.sgv[0]); - console.log('download_timestamp', packet.download_timestamp, Date.parse(packet.download_timestamp)); - console.log("WRITE TO MONGO"); - createProtoStream(packet).pipe(core.persist(function empty(err, result) { - console.log("DONE WRITING TO MONGO", err); - })); - - // core.write(packet); - break; - default: - console.log(topic, 'on message', 'msg', msg); - // core.write(msg); - break; +function toSensor(proto, receiver_time, download_time) { + console.log("Receiver time: ", receiver_time); + console.log("Record time: ", proto.sys_timestamp_sec); + console.log("Download time: ", download_time.unix()); + var record_offset = receiver_time - proto.sys_timestamp_sec; + var record_time = download_time.subtract(record_offset, 'second'); + + console.log("errr", " Offset: ", record_offset, " Record time: ", record_time.format()); + + var obj = { + device: 'dexcom' + , date: record_time.unix() * 1000 + , dateString: record_time.format() + , filtered: proto.filtered + , unfiltered: proto.unfiltered + , rssi: proto.rssi + , type: 'sensor' + }; + return obj; +} + +function createSensorProtoStream(packet, download_time) { + var stream = es.readArray(packet.sensor); + var receiver_time = packet.receiver_system_time_sec; + + function map(item, next) { + var r = toSensor(item, receiver_time, download_time); + console.log("ITEM", item, "TO Sensor", r); + next(null, r); + } + + return stream.pipe(es.map(map)); +} + +function toMeter(proto, receiver_time, download_time) { + console.log("Receiver time: ", receiver_time); + console.log("Record time: ", proto.sys_timestamp_sec); + console.log("Download time: ", download_time.unix()); + var record_offset = receiver_time - proto.sys_timestamp_sec; + var record_time = download_time.subtract(record_offset, 'second'); + + console.log("errr", " Offset: ", record_offset, " Record time: ", record_time.format()); + + var obj = { + device: 'dexcom' + , date: record_time.unix() * 1000 + , dateString: record_time.format() + , mbg: proto.mbg + , type: 'mbg' + }; + return obj; +} + +function createMeterProtoStream(packet, download_time) { + var stream = es.readArray(packet.meter); + var receiver_time = packet.receiver_system_time_sec; + + function map(item, next) { + var r = toMeter(item, receiver_time, download_time); + console.log("ITEM", item, "TO Meter", r); + next(null, r); + } + + return stream.pipe(es.map(map)); +} + +function configure(env, core, devicestatus) { + var uri = env['MQTT_MONITOR']; + var opts = { + encoding: 'binary', + clean: false, + clientId: 'master' + }; + var client = mqtt.connect(uri, opts); + var downloads = downloader(); + client.subscribe('sgvs'); + client.subscribe('published'); + client.subscribe('/downloads/protobuf', {qos: 2}, granted); + client.subscribe('/uploader', granted); + client.subscribe('/entries/sgv', granted); + function granted() { + console.log('granted', arguments); } - }); - client.entries = process(client); - client.every = every; - return client; + + + client.on('message', function (topic, msg) { + console.log('topic', topic); + console.log(topic, 'on message', 'msg', msg.length); + switch (topic) { + case '/uploader': + console.log({type: topic, msg: msg.toString()}); + break; + case '/downloads/protobuf': + var b = new Buffer(msg, 'binary'); + console.log("BINARY", b.length, b.toString('hex')); + try { + var packet = downloads.parse(b); + if (!packet.type) { + packet.type = topic; + } + } catch (e) { + console.log("DID NOT PARSE", e); + break; + } + console.log('DOWNLOAD msg', msg.length, packet); + console.log('download SGV', packet.sgv[0]); + console.log('download_timestamp', packet.download_timestamp, Date.parse(packet.download_timestamp)); + console.log("WRITE TO MONGO"); + var download_timestamp = moment(packet.download_timestamp); + createProtoStream(packet, download_timestamp).pipe(core.persist(function empty(err, result) { + console.log("DONE WRITING SGV TO MONGO", result); + })); + createCalProtoStream(packet, download_timestamp).pipe(core.persist(function empty(err, result) { + console.log("DONE WRITING Cal TO MONGO", result); + })); + createMeterProtoStream(packet, download_timestamp).pipe(core.persist(function empty(err, result) { + console.log("DONE WRITING Meter TO MONGO", result); + })); + createSensorProtoStream(packet, download_timestamp).pipe(core.persist(function empty(err, result) { + console.log("DONE WRITING Sensor TO MONGO", err); + })); + devicestatus.create( { uploaderBattery: packet.uploader_battery, created_at: download_timestamp.toISOString() } , function empty(err, result) { + console.log("DONE WRITING TO MONGO devicestatus ", result, err); + }); + + // core.write(packet); + break; + default: + console.log(topic, 'on message', 'msg', msg); + // core.write(msg); + break; + } + }); + client.entries = process(client); + client.every = every; + return client; } module.exports = configure; diff --git a/server.js b/server.js index 374cde658fa..9b605dfaab1 100644 --- a/server.js +++ b/server.js @@ -83,7 +83,7 @@ store(function ready ( ) { console.log('listening', PORT); if (env.MQTT_MONITOR) { - var mqtt = require('./lib/mqtt')(env, entries); + var mqtt = require('./lib/mqtt')(env, entries, devicestatus); var es = require('event-stream'); es.pipeline(mqtt.entries, entries.map( ), mqtt.every(entries)); } From b37069eedbdf2a88e02ce9a1e9de05499e24416a Mon Sep 17 00:00:00 2001 From: Ben West Date: Wed, 31 Dec 2014 13:13:06 -0500 Subject: [PATCH 017/205] allow multiple instances of mqtt to multiplex This is intended to allow multiple listeners on mqtt to each receive data from MQTT, so long as they are writing to different databases. --- env.js | 3 +++ lib/mqtt.js | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/env.js b/env.js index 6d7c1830a1c..c94d0d64e02 100644 --- a/env.js +++ b/env.js @@ -34,6 +34,9 @@ function config ( ) { env.PORT = readENV('PORT', 1337); env.mongo = readENV('MONGO_CONNECTION') || readENV('MONGO') || readENV('MONGOLAB_URI'); env.mongo_collection = readENV('MONGO_COLLECTION', 'entries'); + if (env.MQTT_MONITOR) { + env.mqtt_client_id = [env.mongo.split('/').pop( ), env.mongo_collection].join('.'); + } env.settings_collection = readENV('MONGO_SETTINGS_COLLECTION', 'settings'); env.treatments_collection = readENV('MONGO_TREATMENTS_COLLECTION', 'treatments'); env.devicestatus_collection = readENV('MONGO_DEVICESTATUS_COLLECTION', 'devicestatus'); diff --git a/lib/mqtt.js b/lib/mqtt.js index 1d088ceffc7..bda036d73f2 100644 --- a/lib/mqtt.js +++ b/lib/mqtt.js @@ -80,7 +80,7 @@ function configure (env, core) { var opts = { encoding: 'binary', clean: false, - clientId: 'master' + clientId: env.mqtt_client_id }; var client = mqtt.connect(uri, opts); var downloads = downloader( ); From 76b9a155819588607341690018e67e7fbed5b115 Mon Sep 17 00:00:00 2001 From: Kevin Lee Date: Wed, 31 Dec 2014 18:52:36 -0600 Subject: [PATCH 018/205] Store the download object in the entries for debugging purposes Converting Sensor filtered and unfiltered data to a proper number --- lib/mqtt.js | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/mqtt.js b/lib/mqtt.js index c411d659979..544711386b6 100644 --- a/lib/mqtt.js +++ b/lib/mqtt.js @@ -123,8 +123,8 @@ function toSensor(proto, receiver_time, download_time) { device: 'dexcom' , date: record_time.unix() * 1000 , dateString: record_time.format() - , filtered: proto.filtered - , unfiltered: proto.unfiltered + , filtered: new Long(proto.filtered).toInt() + , unfiltered: new Long(proto.unfiltered).toInt() , rssi: proto.rssi , type: 'sensor' }; @@ -231,10 +231,19 @@ function configure(env, core, devicestatus) { createSensorProtoStream(packet, download_timestamp).pipe(core.persist(function empty(err, result) { console.log("DONE WRITING Sensor TO MONGO", err); })); - devicestatus.create( { uploaderBattery: packet.uploader_battery, created_at: download_timestamp.toISOString() } , function empty(err, result) { + packet.type = "download"; + devicestatus.create({ + uploaderBattery: packet.uploader_battery, + created_at: download_timestamp.toISOString() + }, function empty(err, result) { console.log("DONE WRITING TO MONGO devicestatus ", result, err); }); + core.create([ packet ], function empty(err, res) { + console.log("Download written to mongo: ", packet) + }); + + // core.write(packet); break; default: From 11a7d9d687665df3e0e0749139917dee497118f0 Mon Sep 17 00:00:00 2001 From: Kevin Lee Date: Wed, 31 Dec 2014 23:04:29 -0600 Subject: [PATCH 019/205] Updating model and removing dead code --- lib/mqtt.js | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/mqtt.js b/lib/mqtt.js index 544711386b6..81378b94847 100644 --- a/lib/mqtt.js +++ b/lib/mqtt.js @@ -26,7 +26,7 @@ function every(storage) { function downloader() { var opts = { - model: decoders.models.CookieMonsterDownload + model: decoders.models.G4Download , json: function (o) { return o; } @@ -37,11 +37,6 @@ function downloader() { return decoders(opts); } -function ReceiverTime(ts) { - var base = Date.parse('2009-01-01T00:00:00-0800'); - return new Date(base + (ts * 1000)); -} - function toSGV(proto, receiver_time, download_time) { console.log("Receiver time: ", receiver_time); console.log("Record time: ", proto.sys_timestamp_sec); From 6b4041a4f825f3b561bddb05ac954640d3b91d9b Mon Sep 17 00:00:00 2001 From: Ben West Date: Fri, 9 Jan 2015 14:38:25 -0800 Subject: [PATCH 020/205] fix mbg undefined --- lib/mqtt.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mqtt.js b/lib/mqtt.js index bbee24d08e5..be6378b3b5b 100644 --- a/lib/mqtt.js +++ b/lib/mqtt.js @@ -152,7 +152,7 @@ function toMeter(proto, receiver_time, download_time) { device: 'dexcom' , date: record_time.unix() * 1000 , dateString: record_time.format() - , mbg: proto.mbg + , mbg: proto.mbg || proto.meter_bg_mgdl , type: 'mbg' }; return obj; From ddc9bf2cb4e86b828764b7eb459b889f85d9faa7 Mon Sep 17 00:00:00 2001 From: Ben West Date: Fri, 9 Jan 2015 15:32:54 -0800 Subject: [PATCH 021/205] only need one stream factory Pass around the sync function for less code. --- lib/mqtt.js | 191 +++++++++++++++++----------------------------------- 1 file changed, 60 insertions(+), 131 deletions(-) diff --git a/lib/mqtt.js b/lib/mqtt.js index be6378b3b5b..b7c81d88a57 100644 --- a/lib/mqtt.js +++ b/lib/mqtt.js @@ -37,137 +37,62 @@ function downloader() { return decoders(opts); } -function toSGV(proto, receiver_time, download_time) { - console.log("Receiver time: ", receiver_time); - console.log("Record time: ", proto.sys_timestamp_sec); - console.log("Download time: ", download_time.unix()); - var record_offset = receiver_time - proto.sys_timestamp_sec; - var record_time = download_time.clone().subtract(record_offset, 'second'); - - console.log("errr", " Offset: ", record_offset, " Record time: ", record_time.format()); - - var obj = { - device: 'dexcom' - , date: record_time.unix() * 1000 - , dateString: record_time.format() - , sgv: proto.sgv_mgdl - , direction: direction(proto.trend) - , noise: proto.noise - , type: 'sgv' - }; - return obj; +function toSGV (proto, vars) { + vars.sgv = proto.sgv_mgdl; + vars.direction = direction(proto.trend); + vars.noise = proto.noise; + vars.type = 'sgv'; + return vars; } -function createProtoStream(packet, download_time) { - var stream = es.readArray(packet.sgv); - var receiver_time = packet.receiver_system_time_sec; - - function map(item, next) { - var r = toSGV(item, receiver_time, download_time); - console.log("ITEM", item, "TO SGV", r); - next(null, r); - } - - return stream.pipe(es.map(map)); +function toCal (proto, vars) { + vars.slope = proto.slope; + vars.intercept = proto.intercept; + vars.scale = proto.scale; + vars.type = 'cal'; + return vars; } -function toCal(proto, receiver_time, download_time) { - console.log("Receiver time: ", receiver_time); - console.log("Record time: ", proto.sys_timestamp_sec); - console.log("Download time: ", download_time.unix()); - var record_offset = receiver_time - proto.sys_timestamp_sec; - var record_time = download_time.clone().subtract(record_offset, 'second'); - - console.log("errr", " Offset: ", record_offset, " Record time: ", record_time.format()); - - var obj = { - device: 'dexcom' - , date: record_time.unix() * 1000 - , dateString: record_time.format() - , slope: proto.slope - , intercept: proto.intercept - , scale: proto.scale - , type: 'cal' - }; - return obj; +function toSensor (proto, vars) { + vars.filtered = new Long(proto.filtered).toInt(); + vars.unfiltered = new Long(proto.unfiltered).toInt(); + vars.rssi = proto.rssi; + vars.type = 'sensor'; + return vars; } -function createCalProtoStream(packet, download_time) { - var stream = es.readArray(packet.cal); - var receiver_time = packet.receiver_system_time_sec; - - function map(item, next) { - var r = toCal(item, receiver_time, download_time); - console.log("ITEM", item, "TO CAL", r); - next(null, r); - } - - return stream.pipe(es.map(map)); +function toMeter (proto, result) { + result.type = 'mbg'; + result.mbg = proto.mbg || proto.meter_bg_mgdl; + return result; } -function toSensor(proto, receiver_time, download_time) { - console.log("Receiver time: ", receiver_time); - console.log("Record time: ", proto.sys_timestamp_sec); - console.log("Download time: ", download_time.unix()); - var record_offset = receiver_time - proto.sys_timestamp_sec; - var record_time = download_time.clone().subtract(record_offset, 'second'); - - console.log("errr", " Offset: ", record_offset, " Record time: ", record_time.format()); - - var obj = { - device: 'dexcom' - , date: record_time.unix() * 1000 - , dateString: record_time.format() - , filtered: new Long(proto.filtered).toInt() - , unfiltered: new Long(proto.unfiltered).toInt() - , rssi: proto.rssi - , type: 'sensor' - }; - return obj; -} - -function createSensorProtoStream(packet, download_time) { - var stream = es.readArray(packet.sensor); - var receiver_time = packet.receiver_system_time_sec; - - function map(item, next) { - var r = toSensor(item, receiver_time, download_time); - console.log("ITEM", item, "TO Sensor", r); - next(null, r); - } - - return stream.pipe(es.map(map)); +function toTimestamp (proto, receiver_time, download_time) { + var record_offset = receiver_time - proto.sys_timestamp_sec; + var record_time = download_time.clone( ).subtract(record_offset, 'second'); + var obj = { + device: 'dexcom' + , date: record_time.unix() * 1000 + , dateString: record_time.format( ) + }; + return obj; } -function toMeter(proto, receiver_time, download_time) { - console.log("Receiver time: ", receiver_time); - console.log("Record time: ", proto.sys_timestamp_sec); - console.log("Download time: ", download_time.unix()); - var record_offset = receiver_time - proto.sys_timestamp_sec; - var record_time = download_time.clone().subtract(record_offset, 'second'); - - console.log("errr", " Offset: ", record_offset, " Record time: ", record_time.format()); - - var obj = { - device: 'dexcom' - , date: record_time.unix() * 1000 - , dateString: record_time.format() - , mbg: proto.mbg || proto.meter_bg_mgdl - , type: 'mbg' - }; - return obj; -} - -function createMeterProtoStream(packet, download_time) { - var stream = es.readArray(packet.meter); - var receiver_time = packet.receiver_system_time_sec; - +function iter_mqtt_record_stream (packet, prop, sync) { + var list = packet[prop]; + console.log('incoming', prop, (list || [ ]).length); + var stream = es.readArray(list || [ ]); + var receiver_time = packet.receiver_system_time_sec; + var download_time = moment(packet.download_timestamp); function map(item, next) { - var r = toMeter(item, receiver_time, download_time); - console.log("ITEM", item, "TO Meter", r); + var timestamped = toTimestamp(item, receiver_time, download_time.clone( )); + var r = sync(item, timestamped); + if (!('type' in r)) { + r.type = prop; + } + console.log("ITEM", item, "TO", prop, r); next(null, r); } - return stream.pipe(es.map(map)); } @@ -211,22 +136,26 @@ function configure(env, core, devicestatus) { } console.log('DOWNLOAD msg', msg.length, packet); console.log('download SGV', packet.sgv[0]); - console.log('download_timestamp', packet.download_timestamp, Date.parse(packet.download_timestamp)); + console.log('download_timestamp', packet.download_timestamp, new Date(Date.parse(packet.download_timestamp))); console.log("WRITE TO MONGO"); var download_timestamp = moment(packet.download_timestamp); if (packet.download_status == 0) { - createProtoStream(packet, download_timestamp).pipe(core.persist(function empty(err, result) { - console.log("DONE WRITING SGV TO MONGO", result); - })); - createCalProtoStream(packet, download_timestamp).pipe(core.persist(function empty(err, result) { - console.log("DONE WRITING Cal TO MONGO", result); - })); - createMeterProtoStream(packet, download_timestamp).pipe(core.persist(function empty(err, result) { - console.log("DONE WRITING Meter TO MONGO", result); - })); - createSensorProtoStream(packet, download_timestamp).pipe(core.persist(function empty(err, result) { - console.log("DONE WRITING Sensor TO MONGO", err); - })); + iter_mqtt_record_stream(packet, 'sgv', toSGV) + .pipe(core.persist(function empty(err, result) { + console.log("DONE WRITING SGV TO MONGO", err, result.length); + })); + iter_mqtt_record_stream(packet, 'cal', toCal) + .pipe(core.persist(function empty(err, result) { + console.log("DONE WRITING Cal TO MONGO", err, result.length); + })); + iter_mqtt_record_stream(packet, 'meter', toMeter) + .pipe(core.persist(function empty(err, result) { + console.log("DONE WRITING Meter TO MONGO", err, result.length); + })); + iter_mqtt_record_stream(packet, 'sensor', toSensor) + .pipe(core.persist(function empty(err, result) { + console.log("DONE WRITING Sensor TO MONGO", err, result.length); + })); } packet.type = "download"; devicestatus.create({ From de7cb78dcafb51f01d44bd0aaba65ad626d64cbf Mon Sep 17 00:00:00 2001 From: Ben West Date: Fri, 9 Jan 2015 15:43:25 -0800 Subject: [PATCH 022/205] force strict zero check --- lib/mqtt.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mqtt.js b/lib/mqtt.js index b7c81d88a57..8f2a9fc6f57 100644 --- a/lib/mqtt.js +++ b/lib/mqtt.js @@ -139,7 +139,7 @@ function configure(env, core, devicestatus) { console.log('download_timestamp', packet.download_timestamp, new Date(Date.parse(packet.download_timestamp))); console.log("WRITE TO MONGO"); var download_timestamp = moment(packet.download_timestamp); - if (packet.download_status == 0) { + if (packet.download_status === 0) { iter_mqtt_record_stream(packet, 'sgv', toSGV) .pipe(core.persist(function empty(err, result) { console.log("DONE WRITING SGV TO MONGO", err, result.length); From a3c968278c1cc50e19df42aa26dd0d7dc25bcaed Mon Sep 17 00:00:00 2001 From: Douglas Eichelberger Date: Sat, 14 Feb 2015 17:01:24 -0800 Subject: [PATCH 023/205] fix coveralls badge branch --- CONTRIBUTING.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dbac299d8e1..c63417fa918 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,15 +12,14 @@ [build-url]: https://travis-ci.org/nightscout/cgm-remote-monitor [dependency-img]: https://img.shields.io/david/nightscout/cgm-remote-monitor.svg [dependency-url]: https://david-dm.org/nightscout/cgm-remote-monitor -[coverage-img]: https://img.shields.io/coveralls/nightscout/cgm-remote-monitor/coverage.svg -[coverage-url]: https://coveralls.io/r/nightscout/cgm-remote-monitor?branch=dev +[coverage-img]: https://img.shields.io/coveralls/nightscout/cgm-remote-monitor/master.svg +[coverage-url]: https://coveralls.io/r/nightscout/cgm-remote-monitor?branch=master [gitter-img]: https://img.shields.io/badge/Gitter-Join%20Chat%20%E2%86%92-1dce73.svg [gitter-url]: https://gitter.im/nightscout/public [ready-img]: https://badge.waffle.io/nightscout/cgm-remote-monitor.svg?label=ready&title=Ready [waffle]: https://waffle.io/nightscout/cgm-remote-monitor [progress-img]: https://badge.waffle.io/nightscout/cgm-remote-monitor.svg?label=in+progress&title=In+Progress - ## Design Participate in the design process by creating an issue to discuss your From ecdabe815d1b124af5db31fbed3a22f22d148c40 Mon Sep 17 00:00:00 2001 From: Douglas Eichelberger Date: Sat, 14 Feb 2015 10:48:09 -0800 Subject: [PATCH 024/205] overhaul Makefile and .travis.yml add tests for node v0.12 move coveralls to after_script (prevents coveralls from failing travis) use project installs of mocha and istanbul exclusively resolves #414 --- .travis.yml | 18 ++++++------------ Makefile | 40 ++++++++++++++++++++++++++++++---------- package.json | 3 +-- 3 files changed, 37 insertions(+), 24 deletions(-) diff --git a/.travis.yml b/.travis.yml index 99660614597..562c8d9edbb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,8 @@ language: node_js +sudo: false node_js: - - "0.10" - - "0.11" -matrix: - allow_failures: - - node_js: "0.11" -services: - - mongodb -before_script: - - sleep 10 - - echo mongo mongo_travis -script: - - make travis + - "0.10" + - "0.12" +services: mongodb +script: make travis +after_script: make report diff --git a/Makefile b/Makefile index 15dc02705ff..43db4909df3 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ +# Nightscout tests/builds/analysis TESTS=tests/*.js MONGO_CONNECTION?=mongodb://localhost/test_db CUSTOMCONNSTR_mongo_settings_collection?=test_settings @@ -7,19 +8,38 @@ MONGO_SETTINGS=MONGO_CONNECTION=${MONGO_CONNECTION} \ CUSTOMCONNSTR_mongo_collection=${CUSTOMCONNSTR_mongo_collection} \ CUSTOMCONNSTR_mongo_settings_collection=${CUSTOMCONNSTR_mongo_settings_collection} +# XXX.bewest: Mocha is an odd process, and since things are being +# wrapped and transformed, this odd path needs to be used, not the +# normal wrapper. When ./node_modules/.bin/mocha is used, no coverage +# information is generated. This happens because typical shell +# wrapper performs process management that mucks with the test +# coverage reporter's ability to instrument the tests correctly. +# Hard coding it to the local with our pinned version is bigger for +# initial installs, but ensures a consistent environment everywhere. +# On Travis, ./node_modules/.bin and other `nvm` and `npm` bundles are +# inserted into the default `$PATH` enviroinment, making pointing to +# the unwrapped mocha executable necessary. +MOCHA=./node_modules/mocha/bin/_mocha +# Pinned from dependency list. +ISTANBUL=./node_modules/.bin/istanbul +ANALYZED=./coverage/lcov.info + all: test -travis-cov: - NODE_ENV=test \ - ${MONGO_SETTINGS} \ - istanbul cover ./node_modules/mocha/bin/_mocha --report lcovonly -- -vvv -R tap ${TESTS} && \ - cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js && \ - rm -rf ./coverage +coverage: + NODE_ENV=test ${MONGO_SETTINGS} \ + ${ISTANBUL} cover ${MOCHA} -- -R tap ${TESTS} + +report: + test -f ${ANALYZED} && \ + (npm install coveralls && cat ${ANALYZED} | \ + ./node_modules/.bin/coveralls) || echo "NO COVERAGE" test: - ${MONGO_SETTINGS} \ - mocha --verbose -vvv -R tap ${TESTS} + ${MONGO_SETTINGS} ${MOCHA} -R tap ${TESTS} -travis: test travis-cov +travis: + NODE_ENV=test ${MONGO_SETTINGS} \ + ${ISTANBUL} cover ${MOCHA} --report lcovonly -- -R tap ${TESTS} -.PHONY: test +.PHONY: all coverage report test travis diff --git a/package.json b/package.json index e5eede089c6..e3e32297f36 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "postinstall": "node node_modules/bower/bin/bower install" }, "engines": { - "node": "~0.10.0" + "node": "0.10.x" }, "dependencies": { "body-parser": "^1.4.3", @@ -46,7 +46,6 @@ "git-rev": "git://github.com/bewest/git-rev.git" }, "devDependencies": { - "coveralls": "~2.11.2", "istanbul": "~0.3.5", "mocha": "~1.20.1", "should": "~4.0.4", From 64ad3f7c9a376d29bd970dea6a80f69cffc31231 Mon Sep 17 00:00:00 2001 From: Douglas Eichelberger Date: Tue, 24 Feb 2015 21:23:13 -0800 Subject: [PATCH 025/205] fix environmental variable example The values of environmental variables should not be in quotes. This change also adds MONGO_COLLECTION to the example. --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b05a49146a7..69b1e5240a7 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,8 @@ Use the [autoconfigure tool][autoconfigure] to sync an uploader to your config. Easy to emulate on the commandline: ```bash - echo 'MONGO_CONNECTION="mongodb://sally:sallypass@ds099999.mongolab.com:99999/nightscout"' >> my.env + echo 'MONGO_CONNECTION=mongodb://sally:sallypass@ds099999.mongolab.com:99999/nightscout' >> my.env + echo 'MONGO_COLLECTION=entries' >> my.env ``` From now on you can run using From ea1dbccdb8f8b84c0f35603d818d07219a74e885 Mon Sep 17 00:00:00 2001 From: Jason Calabrese Date: Thu, 25 Sep 2014 00:57:33 -0700 Subject: [PATCH 026/205] forgot to git add some profile api files --- lib/api/profile/index.js | 47 ++++++++++++++++++++++++++++++++++++++++ lib/profile.js | 24 ++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 lib/api/profile/index.js create mode 100644 lib/profile.js diff --git a/lib/api/profile/index.js b/lib/api/profile/index.js new file mode 100644 index 00000000000..926bbdfdca7 --- /dev/null +++ b/lib/api/profile/index.js @@ -0,0 +1,47 @@ +'use strict'; + +var consts = require('../../constants'); + +function configure (app, wares, profile) { + var express = require('express'), + api = express.Router( ); + + // invoke common middleware + api.use(wares.sendJSONStatus); + // text body types get handled as raw buffer stream + api.use(wares.bodyParser.raw( )); + // json body types get handled as parsed json + api.use(wares.bodyParser.json( )); + // also support url-encoded content-type + api.use(wares.bodyParser.urlencoded({ extended: true })); + + // List settings available + api.get('/profile/', function(req, res) { + profile.list(function (err, attribute) { + return res.json(attribute); + }); + }); + + function config_authed (app, api, wares, profile) { + + api.post('/profile/', /*TODO: auth disabled for now, need to get login figured out... wares.verifyAuthorization, */ function(req, res) { + var attribute = req.body; + profile.create(attribute, function (err, created) { + if (err) + res.sendJSONStatus(res, consts.HTTP_INTERNAL_ERROR, 'Mongo Error', err); + else + res.json(created); + }); + }); + + } + + if (app.enabled('api')) { + config_authed(app, api, wares, profile); + } + + return api; +} + +module.exports = configure; + diff --git a/lib/profile.js b/lib/profile.js new file mode 100644 index 00000000000..c7ba0e101f9 --- /dev/null +++ b/lib/profile.js @@ -0,0 +1,24 @@ +'use strict'; + +function configure (collection, storage) { + + function create (obj, fn) { + obj.created_at = (new Date( )).toISOString( ); + api( ).insert(obj, function (err, doc) { + fn(null, doc); + }); + } + + function list (fn) { + return api( ).find({ }).sort({created_at: -1}).toArray(fn); + } + + function api ( ) { + return storage.pool.db.collection(collection); + } + + api.list = list; + api.create = create; + return api; +} +module.exports = configure; From 7653ef5b205f27ea9a7f85f901f0cc3958758a86 Mon Sep 17 00:00:00 2001 From: Jason Calabrese Date: Thu, 25 Sep 2014 01:09:55 -0700 Subject: [PATCH 027/205] no POSTs yet --- lib/api/profile/index.js | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/lib/api/profile/index.js b/lib/api/profile/index.js index 926bbdfdca7..5dae6a971ee 100644 --- a/lib/api/profile/index.js +++ b/lib/api/profile/index.js @@ -22,24 +22,6 @@ function configure (app, wares, profile) { }); }); - function config_authed (app, api, wares, profile) { - - api.post('/profile/', /*TODO: auth disabled for now, need to get login figured out... wares.verifyAuthorization, */ function(req, res) { - var attribute = req.body; - profile.create(attribute, function (err, created) { - if (err) - res.sendJSONStatus(res, consts.HTTP_INTERNAL_ERROR, 'Mongo Error', err); - else - res.json(created); - }); - }); - - } - - if (app.enabled('api')) { - config_authed(app, api, wares, profile); - } - return api; } From 6080f6781683c33faffe65efc274d1ee869970f0 Mon Sep 17 00:00:00 2001 From: Jason Calabrese Date: Thu, 25 Sep 2014 00:55:13 -0700 Subject: [PATCH 028/205] more cherry-picking and porting of the profile api/storage from iob-cob --- env.js | 1 + lib/api/index.js | 3 ++- lib/websocket.js | 25 +++++++++++++++++++++---- server.js | 5 +++-- static/js/client.js | 7 ++++--- 5 files changed, 31 insertions(+), 10 deletions(-) diff --git a/env.js b/env.js index 0218d69632a..c063c2274a2 100644 --- a/env.js +++ b/env.js @@ -41,6 +41,7 @@ function config ( ) { env.mongo_collection = readENV('MONGO_COLLECTION', 'entries'); env.settings_collection = readENV('MONGO_SETTINGS_COLLECTION', 'settings'); env.treatments_collection = readENV('MONGO_TREATMENTS_COLLECTION', 'treatments'); + env.profile_collection = readENV('MONGO_PROFILE_COLLECTION', 'profile'); env.devicestatus_collection = readENV('MONGO_DEVICESTATUS_COLLECTION', 'devicestatus'); env.enable = readENV('ENABLE'); diff --git a/lib/api/index.js b/lib/api/index.js index 13ccb4c57ce..9943061395f 100644 --- a/lib/api/index.js +++ b/lib/api/index.js @@ -1,6 +1,6 @@ 'use strict'; -function create (env, entries, settings, treatments, devicestatus) { +function create (env, entries, settings, treatments, profile, devicestatus) { var express = require('express'), app = express( ) ; @@ -47,6 +47,7 @@ function create (env, entries, settings, treatments, devicestatus) { app.use('/', require('./entries/')(app, wares, entries)); app.use('/', require('./settings/')(app, wares, settings)); app.use('/', require('./treatments/')(app, wares, treatments)); + app.use('/', require('./profile/')(app, wares, profile)); app.use('/', require('./devicestatus/')(app, wares, devicestatus)); // Status diff --git a/lib/websocket.js b/lib/websocket.js index 49d10bedbc4..6936ef17358 100644 --- a/lib/websocket.js +++ b/lib/websocket.js @@ -1,5 +1,5 @@ -function websocket (env, server, entries, treatments) { +function websocket (env, server, entries, treatments, profiles) { "use strict"; // CONSTANTS var ONE_HOUR = 3600000, @@ -29,6 +29,7 @@ var dir2Char = { treatmentData = [], mbgData = [], calData = [], + profileData = [], patientData = []; function start ( ) { @@ -146,6 +147,7 @@ function update() { cgmData = []; treatmentData = []; mbgData = []; + profileData = []; var earliest_data = now - TWO_DAYS; var q = { find: {"date": {"$gte": earliest_data}} }; entries.list(q, function (err, results) { @@ -188,8 +190,19 @@ function update() { treatment.x = timestamp.getTime(); return treatment; }); - // all done, do loadData - loadData( ); + + profiles.list(function (err, results) { + // There should be only one document in the profile collection with a DIA. If there are multiple, use the last one. + results.forEach(function(element, index, array) { + if (element) { + if (element.dia) { + profileData[0] = element; + } + } + }); + // all done, do loadData + loadData( ); + }); }); }); @@ -237,6 +250,10 @@ function loadData() { }); } + if (profileData) { + var profile = profileData; + } + if (actualCurrent && actualCurrent < 39) errorCode = actualCurrent; var actualLength = actual.length - 1; @@ -271,7 +288,7 @@ function loadData() { // consolidate and send the data to the client var shouldEmit = is_different(actual, predicted, mbg, treatment, cal); - patientData = [actual, predicted, mbg, treatment, cal]; + patientData = [actual, predicted, mbg, treatment, profile, cal]; console.log('patientData', patientData.length); if (shouldEmit) { emitData( ); diff --git a/server.js b/server.js index b1a5dd8b244..7c761c14276 100644 --- a/server.js +++ b/server.js @@ -50,8 +50,9 @@ var express = require('express'); var entriesStorage = entries.storage(env.mongo_collection, store, pushover); var settings = require('./lib/settings')(env.settings_collection, store); var treatmentsStorage = treatments.storage(env.treatments_collection, store, pushover); +var profile = require('./lib/profile')(env.profile_collection, store); var devicestatusStorage = devicestatus.storage(env.devicestatus_collection, store); -var api = require('./lib/api/')(env, entriesStorage, settings, treatmentsStorage, devicestatusStorage); +var api = require('./lib/api/')(env, entriesStorage, settings, treatmentsStorage, profile, devicestatusStorage); var pebble = require('./lib/pebble'); /////////////////////////////////////////////////// @@ -105,7 +106,7 @@ store(function ready ( ) { // setup socket io for data and message transmission /////////////////////////////////////////////////// var websocket = require('./lib/websocket'); - var io = websocket(env, server, entriesStorage, treatmentsStorage); + var io = websocket(env, server, entriesStorage, treatmentsStorage, profile); }); /////////////////////////////////////////////////// diff --git a/static/js/client.js b/static/js/client.js index 3fd39c6757a..c073f00eef9 100644 --- a/static/js/client.js +++ b/static/js/client.js @@ -28,6 +28,7 @@ var app = {}, browserSettings = {}, browserStorage = $.localStorage; , latestUpdateTime , prevSGV , treatments + , profile , cal , padding = { top: 20, right: 10, bottom: 30, left: 10 } , opacity = {current: 1, DAY: 1, NIGHT: 0.5} @@ -124,7 +125,7 @@ var app = {}, browserSettings = {}, browserStorage = $.localStorage; } else if (noise < 2 && browserSettings.showRawbg != 'always') { return 0; } else if (filtered == 0 || sgv < 40) { - console.info("Skipping ratio adjustment for SGV " + sgv); + console.info('Skipping ratio adjustment for SGV ' + sgv); return scale * (unfiltered - intercept) / slope; } else { var ratio = scale * (filtered - intercept) / slope / sgv; @@ -1335,8 +1336,8 @@ var app = {}, browserSettings = {}, browserStorage = $.localStorage; d.created_at = new Date(d.created_at); }); - cal = d[4][d[4].length-1]; - + profile = d[4][0]; + cal = d[5][d[5].length-1]; var temp1 = [ ]; if (cal && showRawBGs()) { From 7f550b8cf5ec4163dc094fff9677717c0763a30e Mon Sep 17 00:00:00 2001 From: Jason Calabrese Date: Sun, 1 Mar 2015 10:56:40 -0800 Subject: [PATCH 029/205] start getting browserify-express hooked up to expose a shared IOB module --- .gitignore | 3 ++ bundle/bundle.source.js | 12 ++++++ lib/iob.js | 81 +++++++++++++++++++++++++++++++++++++++++ package.json | 3 +- server.js | 13 +++++++ static/index.html | 1 + 6 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 bundle/bundle.source.js create mode 100644 lib/iob.js diff --git a/.gitignore b/.gitignore index b8b6cad5141..06a8414200a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ bower_components/ node_modules/ + +bundle/bundle.out.js + .idea/ *.iml my.env diff --git a/bundle/bundle.source.js b/bundle/bundle.source.js new file mode 100644 index 00000000000..e9c8f6e8dca --- /dev/null +++ b/bundle/bundle.source.js @@ -0,0 +1,12 @@ +(function () { + + window.Nightscout = window.Nightscout || {}; + + window.Nightscout = { + iob: require('../lib/iob')() + }; + + console.info("Nightscout bundle ready", window.Nightscout); + +})(); + diff --git a/lib/iob.js b/lib/iob.js new file mode 100644 index 00000000000..c8f48afafd3 --- /dev/null +++ b/lib/iob.js @@ -0,0 +1,81 @@ +'use strict'; + +function iobTotal(treatments, profile, time) { + var iob= 0; + var activity = 0; + if (!treatments) return {}; + if (typeof time === 'undefined') { + time = new Date(); + } + + treatments.forEach(function(treatment) { + if(treatment.created_at < time) { + var tIOB = iobCalc(treatment, profile, time); + if (tIOB && tIOB.iobContrib) iob += tIOB.iobContrib; + if (tIOB && tIOB.activityContrib) activity += tIOB.activityContrib; + } + }); + return { + iob: iob, + activity: activity + }; +} + +function iobCalc(treatment, profile, time) { + + var dia=profile.dia; + var scaleFactor = 3.0/dia; + var peak = 75; + var sens=profile.sens; + var iobContrib, activityContrib; + var t = time; + if (typeof t === 'undefined') { + t = new Date(); + } + + + if (treatment.insulin) { + var bolusTime=new Date(treatment.created_at); + var minAgo=scaleFactor*(t-bolusTime)/1000/60; + + if (minAgo < 0) { + iobContrib=0; + activityContrib=0; + } + if (minAgo < peak) { + var x = minAgo/5+1; + iobContrib=treatment.insulin*(1-0.001852*x*x+0.001852*x); + activityContrib=sens*treatment.insulin*(2/dia/60/peak)*minAgo; + + } + else if (minAgo < 180) { + var x = (minAgo-75)/5; + iobContrib=treatment.insulin*(0.001323*x*x - .054233*x + .55556); + activityContrib=sens*treatment.insulin*(2/dia/60-(minAgo-peak)*2/dia/60/(60*dia-peak)); + } + else { + iobContrib=0; + activityContrib=0; + } + return { + iobContrib: iobContrib, + activityContrib: activityContrib + }; + } else { + return ''; + } +} + +function IOB(opts) { + + var IOB = { + total: iobTotal + }; + + return IOB; + +} + +if (window) window.IOB = IOB; + +module.exports = IOB; \ No newline at end of file diff --git a/package.json b/package.json index af40c524bb0..7521c1ae00d 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,8 @@ "pushover-notifications": "0.2.0", "sgvdata": "0.0.2", "socket.io": "^0.9.17", - "git-rev": "git://github.com/bewest/git-rev.git" + "git-rev": "git://github.com/bewest/git-rev.git", + "browserify-express": "^0.1.4" }, "devDependencies": { "istanbul": "~0.3.5", diff --git a/server.js b/server.js index b1a5dd8b244..d71380199b3 100644 --- a/server.js +++ b/server.js @@ -82,6 +82,19 @@ var staticFiles = express.static(env.static_files, {maxAge: 60 * 60 * 1000}); // serve the static content app.use(staticFiles); +var browserify_express = require('browserify-express'); +var bundle = browserify_express({ + entry: __dirname + '/bundle/bundle.source.js', + watch: __dirname + '/lib/', + mount: '/public/js/bundle.js', + verbose: true, + //minify: true, + bundle_opts: { debug: true }, // enable inline sourcemap on js files + write_file: __dirname + '/bundle/bundle.out.js' +}); + +app.use(bundle); + // Handle errors with express's errorhandler, to display more readable error messages. var errorhandler = require('errorhandler'); //if (process.env.NODE_ENV === 'development') { diff --git a/static/index.html b/static/index.html index d121858dc24..cee26186d5f 100644 --- a/static/index.html +++ b/static/index.html @@ -220,6 +220,7 @@

Nightscout

+ From ba05243dddd53119ca0906b8be95e96be14b79c1 Mon Sep 17 00:00:00 2001 From: Jason Calabrese Date: Sun, 1 Mar 2015 11:24:35 -0800 Subject: [PATCH 030/205] use IOB module to display IOB on th chart --- lib/iob.js | 3 ++- static/js/client.js | 32 ++++++++++++++++++++++++++++---- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/lib/iob.js b/lib/iob.js index c8f48afafd3..1de96b35391 100644 --- a/lib/iob.js +++ b/lib/iob.js @@ -17,6 +17,7 @@ function iobTotal(treatments, profile, time) { }); return { iob: iob, + display: Math.round(iob * 10) / 10, activity: activity }; } @@ -69,7 +70,7 @@ function iobCalc(treatment, profile, time) { function IOB(opts) { var IOB = { - total: iobTotal + calcTotal: iobTotal }; return IOB; diff --git a/static/js/client.js b/static/js/client.js index c073f00eef9..85891eeee7b 100644 --- a/static/js/client.js +++ b/static/js/client.js @@ -106,10 +106,17 @@ var app = {}, browserSettings = {}, browserStorage = $.localStorage; function showRawBGs() { return app.enabledOptions - && app.enabledOptions.indexOf('rawbg' > -1) + && app.enabledOptions.indexOf('rawbg') > -1 && (browserSettings.showRawbg == 'always' || browserSettings.showRawbg == 'noise'); } + function showIOB() { + console.info("app.enabledOptions", app.enabledOptions); + console.info("app.enabledOptions.indexOf('iob')", app.enabledOptions.indexOf('iob' > -1)); + return app.enabledOptions + && app.enabledOptions.indexOf('iob') > -1; + } + function rawIsigToRawBg(entry, cal) { var unfiltered = parseInt(entry.unfiltered) || 0 @@ -358,6 +365,7 @@ var app = {}, browserSettings = {}, browserStorage = $.localStorage; }); if (inRetroMode()) { + var time = new Date(brushExtent[1] - THIRTY_MINS_IN_MS); // filter data for -12 and +5 minutes from reference time for retrospective focus data prediction var lookbackTime = (lookback + 2) * FIVE_MINS_IN_MS + 2 * ONE_MIN_IN_MS; nowData = nowData.filter(function(d) { @@ -379,9 +387,17 @@ var app = {}, browserSettings = {}, browserStorage = $.localStorage; updateCurrentSGV(focusPoint.y); + var details = calcBGDelta(prevfocusPoint.y, focusPoint.y); + + if (showIOB()) { + var iob = Nightscout.iob.calcTotal(treatments, profile, time); + console.info("retro IOB", iob); + details += ", IOB:" + iob.display; + } + currentBG.css('text-decoration','line-through'); currentDirection.html(focusPoint.y < 39 ? '✖' : focusPoint.direction); - currentDetails.text(calcBGDelta(prevfocusPoint.y, focusPoint.y)).css('text-decoration','line-through'); + currentDetails.text(details).css('text-decoration','line-through'); } else { currentBG.text('---').css('text-decoration',''); currentDirection.text('-'); @@ -389,7 +405,7 @@ var app = {}, browserSettings = {}, browserStorage = $.localStorage; } $('#currentTime') - .text(formatTime(new Date(brushExtent[1] - THIRTY_MINS_IN_MS))) + .text(formatTime(time)) .css('text-decoration','line-through'); $('#lastEntry').text('RETRO').removeClass('current'); @@ -402,9 +418,17 @@ var app = {}, browserSettings = {}, browserStorage = $.localStorage; updateClockDisplay(); updateTimeAgo(); + var details = calcBGDelta(prevSGV.y, latestSGV.y); + + if (showIOB()) { + var iob = Nightscout.iob.calcTotal(treatments, profile, nowDate); + console.info("current IOB", iob); + details += ", IOB:" + iob.display; + } + currentBG.css('text-decoration', ''); currentDirection.html(latestSGV.y < 39 ? '✖' : latestSGV.direction); - currentDetails.text(calcBGDelta(prevSGV.y, latestSGV.y)).css('text-decoration',''); + currentDetails.text(details).css('text-decoration',''); } xScale.domain(brush.extent()); From d9f0c4cd10e6e5bb033c4f43290277a746111583 Mon Sep 17 00:00:00 2001 From: Jason Calabrese Date: Sun, 1 Mar 2015 11:31:37 -0800 Subject: [PATCH 031/205] removed debugs --- static/js/client.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/static/js/client.js b/static/js/client.js index 85891eeee7b..9962173e21e 100644 --- a/static/js/client.js +++ b/static/js/client.js @@ -111,8 +111,6 @@ var app = {}, browserSettings = {}, browserStorage = $.localStorage; } function showIOB() { - console.info("app.enabledOptions", app.enabledOptions); - console.info("app.enabledOptions.indexOf('iob')", app.enabledOptions.indexOf('iob' > -1)); return app.enabledOptions && app.enabledOptions.indexOf('iob') > -1; } @@ -391,7 +389,6 @@ var app = {}, browserSettings = {}, browserStorage = $.localStorage; if (showIOB()) { var iob = Nightscout.iob.calcTotal(treatments, profile, time); - console.info("retro IOB", iob); details += ", IOB:" + iob.display; } @@ -422,7 +419,6 @@ var app = {}, browserSettings = {}, browserStorage = $.localStorage; if (showIOB()) { var iob = Nightscout.iob.calcTotal(treatments, profile, nowDate); - console.info("current IOB", iob); details += ", IOB:" + iob.display; } From a693304ab2eaac861589d4072b971b20268e3197 Mon Sep 17 00:00:00 2001 From: Jason Calabrese Date: Sun, 1 Mar 2015 11:48:55 -0800 Subject: [PATCH 032/205] now using new IOB module for /pebble --- lib/iob.js | 9 ++++++--- lib/pebble.js | 25 +++++++++++++++++++++++-- server.js | 4 ++-- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/lib/iob.js b/lib/iob.js index 1de96b35391..38f7674bff7 100644 --- a/lib/iob.js +++ b/lib/iob.js @@ -1,6 +1,11 @@ 'use strict'; function iobTotal(treatments, profile, time) { + + console.info(">>>>treatments:", treatments); + console.info(">>>>profile:", profile); + console.info(">>>>time:", time); + var iob= 0; var activity = 0; if (!treatments) return {}; @@ -9,7 +14,7 @@ function iobTotal(treatments, profile, time) { } treatments.forEach(function(treatment) { - if(treatment.created_at < time) { + if(new Date(treatment.created_at) < time) { var tIOB = iobCalc(treatment, profile, time); if (tIOB && tIOB.iobContrib) iob += tIOB.iobContrib; if (tIOB && tIOB.activityContrib) activity += tIOB.activityContrib; @@ -77,6 +82,4 @@ function IOB(opts) { } -if (window) window.IOB = IOB; - module.exports = IOB; \ No newline at end of file diff --git a/lib/pebble.js b/lib/pebble.js index d8ea2f7161e..427ee2299b9 100644 --- a/lib/pebble.js +++ b/lib/pebble.js @@ -11,6 +11,8 @@ var DIRECTIONS = { , 'RATE OUT OF RANGE': 9 }; +var iob = require("./iob")(); + function directionToTrend (direction) { var trend = 8; if (direction in DIRECTIONS) { @@ -22,6 +24,8 @@ function directionToTrend (direction) { function pebble (req, res) { var ONE_DAY = 24 * 60 * 60 * 1000; var uploaderBattery; + var treatmentResults; + var profileResult; function requestMetric() { var units = req.query.units; @@ -84,6 +88,7 @@ function pebble (req, res) { var bgs = sgvData.slice(0, count); //for compatibility we're keeping battery here, but it would be better somewhere else bgs[0].battery = uploaderBattery ? "" + uploaderBattery : undefined; + bgs[0].iob = iob.calcTotal(treatmentResults.slice(0, 20), profileResult, new Date(now)).display; var result = { status: [ {now:now}], bgs: bgs, cals: calData.slice(0, count) }; res.setHeader('content-type', 'application/json'); @@ -100,12 +105,28 @@ function pebble (req, res) { var earliest_data = Date.now() - ONE_DAY; var q = { find: {"date": {"$gte": earliest_data}} }; - req.entries.list(q, get_latest); + var tq = { find: {"created_at": {"$gte": new Date(earliest_data).toISOString()}} }; + req.treatments.list(tq, function (err, trs) { + treatmentResults = trs; + req.profile.list(function (err, profileResults) { + profileResults.forEach(function(profile) { + if (profile) { + if (profile.dia) { + profileResult = profile; + } + } + }); + req.entries.list(q, get_latest); + }); + }); }); } -function configure (entries, devicestatus, env) { + +function configure (entries, treatments, profile, devicestatus, env) { function middle (req, res, next) { req.entries = entries; + req.treatments = treatments; + req.profile = profile; req.devicestatus = devicestatus; req.rawbg = env.enable && env.enable.indexOf('rawbg') > -1; next( ); diff --git a/server.js b/server.js index 8ea35abe1ea..be6cf522218 100644 --- a/server.js +++ b/server.js @@ -72,7 +72,7 @@ app.enable('trust proxy'); // Allows req.secure test on heroku https connections app.use('/api/v1', api); // pebble data -app.get('/pebble', pebble(entriesStorage, devicestatusStorage, env)); +app.get('/pebble', pebble(entriesStorage, treatmentsStorage, profile, devicestatusStorage, env)); //app.get('/package.json', software); @@ -86,7 +86,7 @@ app.use(staticFiles); var browserify_express = require('browserify-express'); var bundle = browserify_express({ entry: __dirname + '/bundle/bundle.source.js', - watch: __dirname + '/lib/', + watch: [__dirname + '/lib/', __dirname + '/bundle/bundle.source.js'], mount: '/public/js/bundle.js', verbose: true, //minify: true, From 85a19c765fc282ba4a132ea9fe7bb8914f1c4bcf Mon Sep 17 00:00:00 2001 From: Jason Calabrese Date: Sun, 1 Mar 2015 11:50:16 -0800 Subject: [PATCH 033/205] clean up --- lib/iob.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/iob.js b/lib/iob.js index 38f7674bff7..2c8a38eb181 100644 --- a/lib/iob.js +++ b/lib/iob.js @@ -2,10 +2,6 @@ function iobTotal(treatments, profile, time) { - console.info(">>>>treatments:", treatments); - console.info(">>>>profile:", profile); - console.info(">>>>time:", time); - var iob= 0; var activity = 0; if (!treatments) return {}; @@ -22,7 +18,7 @@ function iobTotal(treatments, profile, time) { }); return { iob: iob, - display: Math.round(iob * 10) / 10, + display: Math.round(iob * 100) / 100, activity: activity }; } From 5a13393d552a085077213ce9b9e25fa5de11f99b Mon Sep 17 00:00:00 2001 From: Jason Calabrese Date: Sun, 1 Mar 2015 12:02:41 -0800 Subject: [PATCH 034/205] only load treatments and profile if iob is enabled --- lib/pebble.js | 47 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/lib/pebble.js b/lib/pebble.js index 427ee2299b9..22d99d6f7b3 100644 --- a/lib/pebble.js +++ b/lib/pebble.js @@ -86,9 +86,11 @@ function pebble (req, res) { var count = parseInt(req.query.count) || 1; var bgs = sgvData.slice(0, count); - //for compatibility we're keeping battery here, but it would be better somewhere else + //for compatibility we're keeping battery and iob here, but they would be better somewhere else bgs[0].battery = uploaderBattery ? "" + uploaderBattery : undefined; - bgs[0].iob = iob.calcTotal(treatmentResults.slice(0, 20), profileResult, new Date(now)).display; + if (req.iob) { + bgs[0].iob = iob.calcTotal(treatmentResults.slice(0, 20), profileResult, new Date(now)).display; + } var result = { status: [ {now:now}], bgs: bgs, cals: calData.slice(0, count) }; res.setHeader('content-type', 'application/json'); @@ -104,24 +106,40 @@ function pebble (req, res) { } var earliest_data = Date.now() - ONE_DAY; - var q = { find: {"date": {"$gte": earliest_data}} }; - var tq = { find: {"created_at": {"$gte": new Date(earliest_data).toISOString()}} }; - req.treatments.list(tq, function (err, trs) { + loadTreatments(req, earliest_data, function (err, trs) { treatmentResults = trs; - req.profile.list(function (err, profileResults) { - profileResults.forEach(function(profile) { - if (profile) { - if (profile.dia) { - profileResult = profile; - } - } - }); - req.entries.list(q, get_latest); + loadProfile(req, function (err, profileResults) { + profileResults.forEach(function(profile) { + if (profile) { + if (profile.dia) { + profileResult = profile; + } + } + }); + var q = { find: {"date": {"$gte": earliest_data}} }; + req.entries.list(q, get_latest); }); }); }); } +function loadTreatments(req, earliest_data, fn) { + if (req.iob) { + var q = { find: {"created_at": {"$gte": new Date(earliest_data).toISOString()}} }; + req.treatments.list(q, fn); + } else { + fn(null, []); + } +} + +function loadProfile(req, fn) { + if (req.iob) { + req.profile.list(fn); + } else { + fn(null, []); + } +} + function configure (entries, treatments, profile, devicestatus, env) { function middle (req, res, next) { req.entries = entries; @@ -129,6 +147,7 @@ function configure (entries, treatments, profile, devicestatus, env) { req.profile = profile; req.devicestatus = devicestatus; req.rawbg = env.enable && env.enable.indexOf('rawbg') > -1; + req.iob = env.enable && env.enable.indexOf('iob') > -1; next( ); } return [middle, pebble]; From ee9dcae87e127ac170bdcce1ed2fb92fc641ce3e Mon Sep 17 00:00:00 2001 From: Jason Calabrese Date: Sun, 1 Mar 2015 12:11:43 -0800 Subject: [PATCH 035/205] more clean up --- lib/iob.js | 119 +++++++++++++++++++++----------------------- static/js/client.js | 4 +- 2 files changed, 60 insertions(+), 63 deletions(-) diff --git a/lib/iob.js b/lib/iob.js index 2c8a38eb181..b9629b2f964 100644 --- a/lib/iob.js +++ b/lib/iob.js @@ -1,80 +1,77 @@ 'use strict'; -function iobTotal(treatments, profile, time) { +function calcTotal(treatments, profile, time) { - var iob= 0; - var activity = 0; - if (!treatments) return {}; - if (typeof time === 'undefined') { - time = new Date(); + var iob = 0 + , activity = 0; + + if (!treatments) return {}; + + if (typeof profile === 'undefined') { + //if there is no profile default to 3 hour dia + profile = {dia: 3, sens: 0}; + } + + if (typeof time === 'undefined') { + time = new Date(); + } + + treatments.forEach(function (treatment) { + if (new Date(treatment.created_at) < time) { + var tIOB = calcTreatment(treatment, profile, time); + if (tIOB && tIOB.iobContrib) iob += tIOB.iobContrib; + if (tIOB && tIOB.activityContrib) activity += tIOB.activityContrib; } + }); - treatments.forEach(function(treatment) { - if(new Date(treatment.created_at) < time) { - var tIOB = iobCalc(treatment, profile, time); - if (tIOB && tIOB.iobContrib) iob += tIOB.iobContrib; - if (tIOB && tIOB.activityContrib) activity += tIOB.activityContrib; - } - }); - return { - iob: iob, - display: Math.round(iob * 100) / 100, - activity: activity - }; + return { + iob: iob, + display: Math.round(iob * 100) / 100, + activity: activity + }; } -function iobCalc(treatment, profile, time) { +function calcTreatment(treatment, profile, time) { - var dia=profile.dia; - var scaleFactor = 3.0/dia; - var peak = 75; - var sens=profile.sens; - var iobContrib, activityContrib; - var t = time; - if (typeof t === 'undefined') { - t = new Date(); - } + var dia = profile.dia + , scaleFactor = 3.0 / dia + , peak = 75 + , sens = profile.sens + , iobContrib = 0 + , activityContrib = 0; + if (treatment.insulin) { + var bolusTime = new Date(treatment.created_at); + var minAgo = scaleFactor * (time - bolusTime) / 1000 / 60; - if (treatment.insulin) { - var bolusTime=new Date(treatment.created_at); - var minAgo=scaleFactor*(t-bolusTime)/1000/60; - - if (minAgo < 0) { - iobContrib=0; - activityContrib=0; - } - if (minAgo < peak) { - var x = minAgo/5+1; - iobContrib=treatment.insulin*(1-0.001852*x*x+0.001852*x); - activityContrib=sens*treatment.insulin*(2/dia/60/peak)*minAgo; - - } - else if (minAgo < 180) { - var x = (minAgo-75)/5; - iobContrib=treatment.insulin*(0.001323*x*x - .054233*x + .55556); - activityContrib=sens*treatment.insulin*(2/dia/60-(minAgo-peak)*2/dia/60/(60*dia-peak)); - } - else { - iobContrib=0; - activityContrib=0; - } - return { - iobContrib: iobContrib, - activityContrib: activityContrib - }; + if (minAgo < peak) { + var x = minAgo / 5 + 1; + iobContrib = treatment.insulin * (1 - 0.001852 * x * x + 0.001852 * x); + activityContrib = sens * treatment.insulin * (2 / dia / 60 / peak) * minAgo; + + } else if (minAgo < 180) { + var x = (minAgo - 75) / 5; + iobContrib = treatment.insulin * (0.001323 * x * x - .054233 * x + .55556); + activityContrib = sens * treatment.insulin * (2 / dia / 60 - (minAgo - peak) * 2 / dia / 60 / (60 * dia - peak)); } else { - return ''; + iobContrib = 0; + activityContrib = 0; } + + } + + return { + iobContrib: iobContrib, + activityContrib: activityContrib + }; + } function IOB(opts) { - var IOB = { - calcTotal: iobTotal - }; - - return IOB; + return { + calcTotal: calcTotal + }; } diff --git a/static/js/client.js b/static/js/client.js index 9962173e21e..0905fe29742 100644 --- a/static/js/client.js +++ b/static/js/client.js @@ -389,7 +389,7 @@ var app = {}, browserSettings = {}, browserStorage = $.localStorage; if (showIOB()) { var iob = Nightscout.iob.calcTotal(treatments, profile, time); - details += ", IOB:" + iob.display; + details += ", IOB: " + iob.display; } currentBG.css('text-decoration','line-through'); @@ -419,7 +419,7 @@ var app = {}, browserSettings = {}, browserStorage = $.localStorage; if (showIOB()) { var iob = Nightscout.iob.calcTotal(treatments, profile, nowDate); - details += ", IOB:" + iob.display; + details += ", IOB: " + iob.display; } currentBG.css('text-decoration', ''); From d7d917b03b0851db0b02e1fd273b1e0f4d624325 Mon Sep 17 00:00:00 2001 From: Douglas Eichelberger Date: Tue, 17 Feb 2015 22:33:16 -0800 Subject: [PATCH 036/205] create all docs before calling back Resolves #423. Also changes the callback signature, invoking it with the error and docs, rather than the error, totalCreated, and docs. The third paramter isn't used, and the second parameter should be an array. (Since totalCreated was 0 on callback, it was falsy and replaced with an empty array when used in format_entries.) --- lib/entries.js | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/lib/entries.js b/lib/entries.js index c8f8eb7a64b..fa83324e304 100644 --- a/lib/entries.js +++ b/lib/entries.js @@ -96,21 +96,23 @@ function storage(name, storage, pushover) { // store new documents using the storage mechanism function create (docs, fn) { - with_collection(function(err, collection) { - if (err) { fn(err); return; } - // potentially a batch insert - var firstErr = null, - totalCreated = 0; - - docs.forEach(function(doc) { - collection.update(doc, doc, {upsert: true}, function (err, created) { - firstErr = firstErr || err; - totalCreated += created; - }); - sendPushover(doc); + with_collection(function(err, collection) { + if (err) { fn(err); return; } + // potentially a batch insert + var firstErr = null, + numDocs = docs.length, + totalCreated = 0; + + docs.forEach(function(doc) { + collection.update(doc, doc, {upsert: true}, function (err, created) { + firstErr = firstErr || err; + if (++totalCreated === numDocs) { + fn(firstErr, docs); + } }); - fn(firstErr, totalCreated, docs); + sendPushover(doc); }); + }); } //currently the Android upload will send the last MBG over and over, make sure we get a single notification From 81a174e00632469a1014b7993c1fc08b3a491615 Mon Sep 17 00:00:00 2001 From: Douglas Eichelberger Date: Tue, 17 Feb 2015 22:52:43 -0800 Subject: [PATCH 037/205] clean up api.entries.test.js --- tests/api.entries.test.js | 55 ++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/tests/api.entries.test.js b/tests/api.entries.test.js index d8a60230b55..f3e368da7a6 100644 --- a/tests/api.entries.test.js +++ b/tests/api.entries.test.js @@ -1,10 +1,10 @@ - var request = require('supertest'); var should = require('should'); var load = require('./fixtures/load'); describe('Entries REST api', function ( ) { var entries = require('../lib/api/entries/'); + before(function (done) { var env = require('../env')( ); this.wares = require('../lib/middleware/')(env); @@ -18,32 +18,39 @@ describe('Entries REST api', function ( ) { self.archive.create(load('json'), done); }); }); + after(function (done) { this.archive( ).remove({ }, done); }); it('should be a module', function ( ) { entries.should.be.ok; - }); - it('/entries.json', function (done) { + + // keep this test pinned at or near the top in order to validate all + // entries successfully uploaded. if res.body.length is short of the + // expected value, it may indicate a regression in the create + // function callback logic in entries.js. + it('gets requested number of entries', function (done) { + var count = 30; request(this.app) - .get('/entries.json') + .get('/entries.json?count=' + count) .expect(200) - .end(function (err, res) { + .end(function (err, res) { // console.log('body', res.body); - res.body.length.should.equal(10); + res.body.length.should.equal(count); done( ); }); }); - it('/entries.json', function (done) { + it('gets default number of entries', function (done) { + var defaultCount = 10; request(this.app) - .get('/entries.json?count=30') + .get('/entries.json') .expect(200) - .end(function (err, res) { + .end(function (err, res) { // console.log('body', res.body); - res.body.length.should.equal(30); + res.body.length.should.equal(defaultCount); done( ); }); }); @@ -52,29 +59,23 @@ describe('Entries REST api', function ( ) { request(this.app) .get('/entries/current.json') .expect(200) - .end(function (err, res) { + .end(function (err, res) { res.body.length.should.equal(1); done( ); // console.log('err', err, 'res', res); }); - }); it('/entries/preview', function (done) { - - request(this.app) - .post('/entries/preview.json') - .send(load('json')) - .expect(201) - .end(function (err, res) { - // console.log(res.body); - res.body.length.should.equal(30); - done( ); - // console.log('err', err, 'res', res); - }) - ; - + request(this.app) + .post('/entries/preview.json') + .send(load('json')) + .expect(201) + .end(function (err, res) { + // console.log(res.body); + res.body.length.should.equal(30); + done( ); + // console.log('err', err, 'res', res); + }); }); - }); - From b946f86f961c4a008a5da88ef688076a3a5f2456 Mon Sep 17 00:00:00 2001 From: Douglas Eichelberger Date: Tue, 17 Feb 2015 22:54:10 -0800 Subject: [PATCH 038/205] prettify example.json --- tests/fixtures/example.json | 213 +++++++++++++++++++++++++++++++++++- 1 file changed, 212 insertions(+), 1 deletion(-) diff --git a/tests/fixtures/example.json b/tests/fixtures/example.json index 88da09c6fa8..aa330937969 100644 --- a/tests/fixtures/example.json +++ b/tests/fixtures/example.json @@ -1 +1,212 @@ -[{"sgv":"5","dateString":"07/19/2014 10:49:15 AM","date":1405792155000,"device":"dexcom","direction":"NOT COMPUTABLE"},{"sgv":"5","dateString":"07/19/2014 10:44:15 AM","date":1405791855000,"device":"dexcom","direction":"NOT COMPUTABLE"},{"sgv":"5","dateString":"07/19/2014 10:39:15 AM","date":1405791555000,"device":"dexcom","direction":"NOT COMPUTABLE"},{"sgv":"5","dateString":"07/19/2014 10:34:15 AM","date":1405791255000,"device":"dexcom","direction":"NOT COMPUTABLE"},{"sgv":"5","dateString":"07/19/2014 10:29:15 AM","date":1405790955000,"device":"dexcom","direction":"NOT COMPUTABLE"},{"sgv":"5","dateString":"07/19/2014 10:24:15 AM","date":1405790655000,"device":"dexcom","direction":"NOT COMPUTABLE"},{"sgv":"5","dateString":"07/19/2014 10:19:15 AM","date":1405790355000,"device":"dexcom","direction":"NOT COMPUTABLE"},{"sgv":"5","dateString":"07/19/2014 10:14:15 AM","date":1405790055000,"device":"dexcom","direction":"NOT COMPUTABLE"},{"sgv":"5","dateString":"07/19/2014 10:09:15 AM","date":1405789755000,"device":"dexcom","direction":"NOT COMPUTABLE"},{"sgv":"5","dateString":"07/19/2014 10:04:15 AM","date":1405789455000,"device":"dexcom","direction":"NOT COMPUTABLE"},{"sgv":"5","dateString":"07/19/2014 09:59:15 AM","date":1405789155000,"device":"dexcom","direction":"NOT COMPUTABLE"},{"sgv":"5","dateString":"07/19/2014 09:54:15 AM","date":1405788855000,"device":"dexcom","direction":"NOT COMPUTABLE"},{"sgv":"178","dateString":"07/19/2014 03:59:15 AM","date":1405767555000,"device":"dexcom","direction":"Flat"},{"sgv":"179","dateString":"07/19/2014 03:54:15 AM","date":1405767255000,"device":"dexcom","direction":"Flat"},{"sgv":"178","dateString":"07/19/2014 03:49:15 AM","date":1405766955000,"device":"dexcom","direction":"Flat"},{"sgv":"177","dateString":"07/19/2014 03:44:15 AM","date":1405766655000,"device":"dexcom","direction":"Flat"},{"sgv":"176","dateString":"07/19/2014 03:39:15 AM","date":1405766355000,"device":"dexcom","direction":"Flat"},{"sgv":"176","dateString":"07/19/2014 03:34:15 AM","date":1405766055000,"device":"dexcom","direction":"Flat"},{"sgv":"175","dateString":"07/19/2014 03:29:16 AM","date":1405765756000,"device":"dexcom","direction":"Flat"},{"sgv":"174","dateString":"07/19/2014 03:24:15 AM","date":1405765455000,"device":"dexcom","direction":"Flat"},{"sgv":"174","dateString":"07/19/2014 03:19:15 AM","date":1405765155000,"device":"dexcom","direction":"Flat"},{"sgv":"175","dateString":"07/19/2014 03:14:15 AM","date":1405764855000,"device":"dexcom","direction":"Flat"},{"sgv":"176","dateString":"07/19/2014 03:09:15 AM","date":1405764555000,"device":"dexcom","direction":"Flat"},{"sgv":"176","dateString":"07/19/2014 03:04:15 AM","date":1405764255000,"device":"dexcom","direction":"Flat"},{"sgv":"173","dateString":"07/19/2014 02:59:15 AM","date":1405763955000,"device":"dexcom","direction":"Flat"},{"sgv":"171","dateString":"07/19/2014 02:54:15 AM","date":1405763655000,"device":"dexcom","direction":"Flat"},{"sgv":"170","dateString":"07/19/2014 02:49:15 AM","date":1405763355000,"device":"dexcom","direction":"Flat"},{"sgv":"171","dateString":"07/19/2014 02:44:15 AM","date":1405763055000,"device":"dexcom","direction":"Flat"},{"sgv":"169","dateString":"07/19/2014 02:39:15 AM","date":1405762755000,"device":"dexcom","direction":"Flat"},{"sgv":"169","dateString":"07/19/2014 02:34:15 AM","date":1405762455000,"device":"dexcom","direction":"Flat"}] \ No newline at end of file +[ + { + "sgv": "5", + "dateString": "07\/19\/2014 10:49:15 AM", + "date": 1405792155000, + "device": "dexcom", + "direction": "NOT COMPUTABLE" + }, + { + "sgv": "5", + "dateString": "07\/19\/2014 10:44:15 AM", + "date": 1405791855000, + "device": "dexcom", + "direction": "NOT COMPUTABLE" + }, + { + "sgv": "5", + "dateString": "07\/19\/2014 10:39:15 AM", + "date": 1405791555000, + "device": "dexcom", + "direction": "NOT COMPUTABLE" + }, + { + "sgv": "5", + "dateString": "07\/19\/2014 10:34:15 AM", + "date": 1405791255000, + "device": "dexcom", + "direction": "NOT COMPUTABLE" + }, + { + "sgv": "5", + "dateString": "07\/19\/2014 10:29:15 AM", + "date": 1405790955000, + "device": "dexcom", + "direction": "NOT COMPUTABLE" + }, + { + "sgv": "5", + "dateString": "07\/19\/2014 10:24:15 AM", + "date": 1405790655000, + "device": "dexcom", + "direction": "NOT COMPUTABLE" + }, + { + "sgv": "5", + "dateString": "07\/19\/2014 10:19:15 AM", + "date": 1405790355000, + "device": "dexcom", + "direction": "NOT COMPUTABLE" + }, + { + "sgv": "5", + "dateString": "07\/19\/2014 10:14:15 AM", + "date": 1405790055000, + "device": "dexcom", + "direction": "NOT COMPUTABLE" + }, + { + "sgv": "5", + "dateString": "07\/19\/2014 10:09:15 AM", + "date": 1405789755000, + "device": "dexcom", + "direction": "NOT COMPUTABLE" + }, + { + "sgv": "5", + "dateString": "07\/19\/2014 10:04:15 AM", + "date": 1405789455000, + "device": "dexcom", + "direction": "NOT COMPUTABLE" + }, + { + "sgv": "5", + "dateString": "07\/19\/2014 09:59:15 AM", + "date": 1405789155000, + "device": "dexcom", + "direction": "NOT COMPUTABLE" + }, + { + "sgv": "5", + "dateString": "07\/19\/2014 09:54:15 AM", + "date": 1405788855000, + "device": "dexcom", + "direction": "NOT COMPUTABLE" + }, + { + "sgv": "178", + "dateString": "07\/19\/2014 03:59:15 AM", + "date": 1405767555000, + "device": "dexcom", + "direction": "Flat" + }, + { + "sgv": "179", + "dateString": "07\/19\/2014 03:54:15 AM", + "date": 1405767255000, + "device": "dexcom", + "direction": "Flat" + }, + { + "sgv": "178", + "dateString": "07\/19\/2014 03:49:15 AM", + "date": 1405766955000, + "device": "dexcom", + "direction": "Flat" + }, + { + "sgv": "177", + "dateString": "07\/19\/2014 03:44:15 AM", + "date": 1405766655000, + "device": "dexcom", + "direction": "Flat" + }, + { + "sgv": "176", + "dateString": "07\/19\/2014 03:39:15 AM", + "date": 1405766355000, + "device": "dexcom", + "direction": "Flat" + }, + { + "sgv": "176", + "dateString": "07\/19\/2014 03:34:15 AM", + "date": 1405766055000, + "device": "dexcom", + "direction": "Flat" + }, + { + "sgv": "175", + "dateString": "07\/19\/2014 03:29:16 AM", + "date": 1405765756000, + "device": "dexcom", + "direction": "Flat" + }, + { + "sgv": "174", + "dateString": "07\/19\/2014 03:24:15 AM", + "date": 1405765455000, + "device": "dexcom", + "direction": "Flat" + }, + { + "sgv": "174", + "dateString": "07\/19\/2014 03:19:15 AM", + "date": 1405765155000, + "device": "dexcom", + "direction": "Flat" + }, + { + "sgv": "175", + "dateString": "07\/19\/2014 03:14:15 AM", + "date": 1405764855000, + "device": "dexcom", + "direction": "Flat" + }, + { + "sgv": "176", + "dateString": "07\/19\/2014 03:09:15 AM", + "date": 1405764555000, + "device": "dexcom", + "direction": "Flat" + }, + { + "sgv": "176", + "dateString": "07\/19\/2014 03:04:15 AM", + "date": 1405764255000, + "device": "dexcom", + "direction": "Flat" + }, + { + "sgv": "173", + "dateString": "07\/19\/2014 02:59:15 AM", + "date": 1405763955000, + "device": "dexcom", + "direction": "Flat" + }, + { + "sgv": "171", + "dateString": "07\/19\/2014 02:54:15 AM", + "date": 1405763655000, + "device": "dexcom", + "direction": "Flat" + }, + { + "sgv": "170", + "dateString": "07\/19\/2014 02:49:15 AM", + "date": 1405763355000, + "device": "dexcom", + "direction": "Flat" + }, + { + "sgv": "171", + "dateString": "07\/19\/2014 02:44:15 AM", + "date": 1405763055000, + "device": "dexcom", + "direction": "Flat" + }, + { + "sgv": "169", + "dateString": "07\/19\/2014 02:39:15 AM", + "date": 1405762755000, + "device": "dexcom", + "direction": "Flat" + }, + { + "sgv": "169", + "dateString": "07\/19\/2014 02:34:15 AM", + "date": 1405762455000, + "device": "dexcom", + "direction": "Flat" + } +] From f7f08fa7d106c12a99bbb253110fe2af3fc8c813 Mon Sep 17 00:00:00 2001 From: Douglas Eichelberger Date: Tue, 17 Feb 2015 23:03:03 -0800 Subject: [PATCH 039/205] fix alignment issues in entries.js --- lib/entries.js | 91 ++++++++++++++++++++++++-------------------------- 1 file changed, 43 insertions(+), 48 deletions(-) diff --git a/lib/entries.js b/lib/entries.js index fa83324e304..4363bf46967 100644 --- a/lib/entries.js +++ b/lib/entries.js @@ -21,7 +21,7 @@ function storage(name, storage, pushover) { with_collection(function (err, collection) { // these functions, find, sort, and limit, are used to // dynamically configure the request, based on the options we've - // been give + // been given // determine find options function find ( ) { @@ -51,13 +51,9 @@ function storage(name, storage, pushover) { // now just stitch them all together limit.call(collection - .find(find( )) - .sort(sort( ))) - // .limit(limit( )) - .toArray(toArray) - ; - // limit.call(sort.call(find.call(collection))).toArray(toArray); - + .find(find( )) + .sort(sort( )) + ).toArray(toArray); }); } @@ -119,56 +115,56 @@ function storage(name, storage, pushover) { var lastMBG = 0; function sendPushover(doc) { - if (doc.type && doc.mbg && doc.type == 'mbg' && doc.date && doc.date != lastMBG && pushover) { - var offset = new Date().getTime() - doc.date; - if (offset > TEN_MINS) { - console.info('No MBG Pushover, offset: ' + offset + ' too big, doc.date: ' + doc.date + ', now: ' + new Date().getTime()); - } else { - var msg = { - expire: 14400, // 4 hours - message: '\nMeter BG: ' + doc.mbg, - title: 'Calibration', - sound: 'magic', - timestamp: new Date(doc.date), - priority: 0, - retry: 30 - }; - - pushover.send(msg, function (err, result) { - console.log(result); - }); - } - lastMBG = doc.date; + if (doc.type && doc.mbg && doc.type == 'mbg' && doc.date && doc.date != lastMBG && pushover) { + var offset = new Date().getTime() - doc.date; + if (offset > TEN_MINS) { + console.info('No MBG Pushover, offset: ' + offset + ' too big, doc.date: ' + doc.date + ', now: ' + new Date().getTime()); + } else { + var msg = { + expire: 14400, // 4 hours + message: '\nMeter BG: ' + doc.mbg, + title: 'Calibration', + sound: 'magic', + timestamp: new Date(doc.date), + priority: 0, + retry: 30 + }; + + pushover.send(msg, function (err, result) { + console.log(result); + }); } + lastMBG = doc.date; + } } function getEntry(fn, id) { - console.info("trying to find entry for id: " + id); - with_collection(function(err, collection) { + console.info("trying to find entry for id: " + id); + with_collection(function(err, collection) { + if (err) + fn(err); + else + collection.findOne({"_id": ObjectID(id)}, function (err, entry) { if (err) - fn(err); + fn(err); else - collection.findOne({"_id": ObjectID(id)}, function (err, entry) { - if (err) - fn(err); - else - fn(null, entry); - }); - }); + fn(null, entry); + }); + }); } function getEntries(fn, count) { - with_collection(function(err, collection) { + with_collection(function(err, collection) { + if (err) + fn(err); + else + collection.find({ }).sort({"date": -1}).limit(count).toArray(function (err, entries) { if (err) - fn(err); + fn(err); else - collection.find({ }).sort({"date": -1}).limit(count).toArray(function (err, entries) { - if (err) - fn(err); - else - fn(null, entries); - }); - }); + fn(null, entries); + }); + }); } // closure to represent the API @@ -204,4 +200,3 @@ module.exports = { storage: storage, ensureIndexes: ensureIndexes }; - From 99a6978dfda1d87d36653a4faaaba4f03cbb1cee Mon Sep 17 00:00:00 2001 From: Douglas Eichelberger Date: Sat, 28 Feb 2015 14:09:41 -0800 Subject: [PATCH 040/205] fix undefined err --- lib/api/entries/index.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/api/entries/index.js b/lib/api/entries/index.js index 34907b5615f..7348311cde6 100644 --- a/lib/api/entries/index.js +++ b/lib/api/entries/index.js @@ -29,7 +29,9 @@ function configure (app, wares, entries) { // Middleware to format any response involving entries. function format_entries (req, res, next) { var output = es.readArray(res.entries || [ ]); - if (res.entries_err) return res.sendJSONStatus(res, consts.HTTP_INTERNAL_ERROR, 'Mongo Error', err); + if (res.entries_err) { + return res.sendJSONStatus(res, consts.HTTP_INTERNAL_ERROR, 'Mongo Error', res.entries_err); + } return res.format({ text: function ( ) { es.pipeline(output, sgvdata.format( ), res); From bccc5e15985af99e9ccc28ed08cc86d329a10a27 Mon Sep 17 00:00:00 2001 From: Douglas Eichelberger Date: Sun, 1 Mar 2015 10:49:21 -0800 Subject: [PATCH 041/205] ignore npm-debug.log --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index b8b6cad5141..a54c7ec4fe4 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ static/bower_components/ # istanbul output coverage/ + +npm-debug.log From 13751dfd4be0d80c2603f1e0f35e127ecd230225 Mon Sep 17 00:00:00 2001 From: Douglas Eichelberger Date: Sun, 1 Mar 2015 11:50:18 -0800 Subject: [PATCH 042/205] rm unnecessary return statements flagged by webstorm: "return is unnecessary as the last statement in a function with no return value" --- lib/api/entries/index.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/api/entries/index.js b/lib/api/entries/index.js index 7348311cde6..87dc532195d 100644 --- a/lib/api/entries/index.js +++ b/lib/api/entries/index.js @@ -115,7 +115,6 @@ function configure (app, wares, entries) { res.entries_err = err; return next( ); }); - return; }, format_entries); api.get('/entries/current', function(req, res, next) { @@ -124,7 +123,6 @@ function configure (app, wares, entries) { res.entries_err = err; return next( ); }); - return; }, format_entries); @@ -133,7 +131,6 @@ function configure (app, wares, entries) { api.post('/entries/preview', function (req, res, next) { req.persist_entries = false; next( ); - return; }, insert_entries, format_entries); if (app.enabled('api')) { @@ -141,7 +138,6 @@ function configure (app, wares, entries) { api.post('/entries/', wares.verifyAuthorization, function (req, res, next) { req.persist_entries = true; next( ); - return; }, insert_entries, format_entries); } From 127e4f5fbbc74b76f6950f464df769246674186e Mon Sep 17 00:00:00 2001 From: Jason Calabrese Date: Sun, 1 Mar 2015 12:29:15 -0800 Subject: [PATCH 043/205] added mocks for treatments and profile so the pebble test doesn't fail --- tests/pebble.test.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/pebble.test.js b/tests/pebble.test.js index 6b3b0743089..d6704ed7928 100644 --- a/tests/pebble.test.js +++ b/tests/pebble.test.js @@ -75,6 +75,18 @@ var entries = { }; //Mock devicestatus +var treatments = { + list: function(callback) { + callback(null, []); + } +}; + +var profile = { + list: function(callback) { + callback(null, []); + } +}; + var devicestatus = { last: function(callback) { callback(null, {uploaderBattery: 100}); @@ -87,7 +99,7 @@ describe('Pebble Endpoint without Raw', function ( ) { var env = require('../env')( ); this.app = require('express')( ); this.app.enable('api'); - this.app.use('/pebble', pebble(entries, devicestatus, env)); + this.app.use('/pebble', pebble(entries, treatments, profile, devicestatus, env)); done(); }); @@ -128,7 +140,7 @@ describe('Pebble Endpoint with Raw', function ( ) { envRaw.enable = "rawbg"; this.appRaw = require('express')( ); this.appRaw.enable('api'); - this.appRaw.use('/pebble', pebbleRaw(entries, devicestatus, envRaw)); + this.appRaw.use('/pebble', pebbleRaw(entries, treatments, profile, devicestatus, envRaw)); done(); }); From 0aea8a7b26ac01cbb913b609f1f2c179cbf0c3d6 Mon Sep 17 00:00:00 2001 From: Jason Calabrese Date: Sun, 1 Mar 2015 13:21:30 -0800 Subject: [PATCH 044/205] very basic IOB test --- lib/iob.js | 2 ++ tests/iob.test.js | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 tests/iob.test.js diff --git a/lib/iob.js b/lib/iob.js index b9629b2f964..97bca5972c5 100644 --- a/lib/iob.js +++ b/lib/iob.js @@ -2,6 +2,8 @@ function calcTotal(treatments, profile, time) { + console.info("trying to calc"); + var iob = 0 , activity = 0; diff --git a/tests/iob.test.js b/tests/iob.test.js new file mode 100644 index 00000000000..e4dad0fcb54 --- /dev/null +++ b/tests/iob.test.js @@ -0,0 +1,33 @@ +var should = require('should'); + +describe('IOB', function ( ) { + var iob = require('../lib/iob')(); + + it('should calculate IOB', function() { + + var time = new Date() + , treatments = [ { + created_at: time - 1, + insulin: "1.00" + } + ] + , profile = { + dia: 3, + sens: 0 + }; + + var rightAfterBolus = iob.calcTotal(treatments, profile, time); + + rightAfterBolus.display.should.equal(1); + + var afterSomeTime = iob.calcTotal(treatments, profile, new Date(time.getTime() + (60 * 60 * 1000))); + + afterSomeTime.display.should.be.lessThan(1); + afterSomeTime.display.should.be.greaterThan(0); + + var afterDIA = iob.calcTotal(treatments, profile, new Date(time.getTime() + (3 * 60 * 60 * 1000))); + + afterDIA.display.should.equal(0); + + }); +}); \ No newline at end of file From aa9233be1bad2c7a51f09ff80bfde3226baecfff Mon Sep 17 00:00:00 2001 From: Jason Calabrese Date: Fri, 6 Mar 2015 00:24:35 -0800 Subject: [PATCH 045/205] new pill display for BG delta and IOB --- static/css/main.css | 58 +++++++++++++++++++++++++++++++++++---------- static/index.html | 4 ++-- static/js/client.js | 26 ++++++++++++-------- 3 files changed, 64 insertions(+), 24 deletions(-) diff --git a/static/css/main.css b/static/css/main.css index ed5ef975b4c..19768f16eaa 100644 --- a/static/css/main.css +++ b/static/css/main.css @@ -68,21 +68,50 @@ body { .bgStatus .currentDetails { font-size: 25%; - text-decoration: line-through; - display: block; - position: relative; - top: -20px; + position: relative; + top: -10px; } -.bgStatus.current .currentBG { - text-decoration: none; + +.currentDetails > span { + border-radius: 5px; + border: 2px solid #808080; + font-size: 80%; + display: inline-block; } -.bgStatus.current .currentDetails { - font-size: 25%; + +#bgButton .currentDetails > span { + border-color: #000; +} + +.currentDetails > span:not(:first-child) { + margin-left: 5px; +} + +.currentDetails > span * { + padding-left: 2px; + padding-right: 2px; + padding-bottom: 1px; +} + +.currentDetails > span em { + font-style: normal; + font-weight: bold; +} + +.currentDetails > span label { + color: #000; + background: #808080; +} + +#bgButton .currentDetails > span label { + color: #808080; + background: inherit; +} + +.bgStatus.current .currentBG { text-decoration: none; - display: block; - position: relative; - top: -10px; } + .currentDirection { font-weight: normal; text-decoration: none; @@ -101,7 +130,7 @@ body { font-size: 25%; } #lastEntry { - background: #999; + background: #808080; border-radius: 5px; color: #000; padding: 0.25em; @@ -261,6 +290,11 @@ div.tooltip { #currentTime { font-size: 85%; } + + .bgStatus .currentDetails { + top: 0px; + } + } @media (max-width: 750px) { diff --git a/static/index.html b/static/index.html index cee26186d5f..703c87db52f 100644 --- a/static/index.html +++ b/static/index.html @@ -30,13 +30,13 @@

Nightscout

--- - - -- +
--
-
 
+
---
- --- +
diff --git a/static/js/client.js b/static/js/client.js index 26a99d453ab..cb534d6dd2d 100644 --- a/static/js/client.js +++ b/static/js/client.js @@ -339,7 +339,7 @@ var app = {}, browserSettings = {}, browserStorage = $.localStorage; bgDeltaString = '+' + bgDelta; } - bgDeltaString = '' + bgDeltaString + '