Separate notifier (#341)

* feat: move worker into separate process
- make db calls flex between notifier worker and 
- wrap rest calls in a separate wrapper
- fix MessageManagermisspelling
- clean up logging
- remove "deleteAfter" functionality for notifications

BREAKING CHANGE: Notifier usage will need a new process and config

* chore: decrease ctx lookup for non-bot scopes

* fix: clean up fissures & arbi's are generated, move data generation into blocking part of script

* task: bot working in parallel

* feat: upgrade twitch notifier

* refactor notifier to use notifiedIds db, clean up add build command, cleaner startup

* fix: close #331, close #300, close #270

* chore: updates

* chore: rebuild lock

* improve twitter

* chore: lint rss embed

.gitignore

@@ -39,6 +39,7 @@ jspm_packages
 
 #ignore prod pm2 config
 genesis.json
+worker.json
 .idea/
 mypm2.json
 

.npmignore

@@ -1,2 +1,5 @@
 # JSDoc
-docs
+docs
+
+.gitignore
+.travis.yaml

package.json

@@ -9,11 +9,14 @@
     "lint": "npx eslint main.js src/",
     "lint:fix": "npx eslint main.js src/ --fix",
     "logs": "pm2 logs genesis",
+    "logs:worker": "pm2 logs worker",
     "test": "npx eslint main.js src/",
     "restart": "pm2 delete genesis && npm start && npm run logs",
     "setup": "npm i",
     "start": "pm2 start genesis.json",
     "start:logs": "npm start && npm run logs",
+    "start:worker": "pm2 start worker.json",
+    "start:logs:worker": "npm run start:worker && pm2 logs worker",
     "stop": "pm2 delete genesis",
     "uninstall": "pm2 kill && rm -rf node_modules/ && npm uninstall -g pm2",
     "update:genesis": "pm2 delete genesis && git checkout -- . && git pull && npm i && npm restart || npm start"
@@ -57,16 +60,17 @@
     "ping": "^0.2.3",
     "rss-feed-emitter": "^3.1.2",
     "sql-template-strings": "^2.2.2",
-    "twitch": "^3.7.1",
+    "twitch": "^4.0.10",
+    "twitch-webhooks": "^4.0.10",
     "url-exists": "^1.0.3",
     "utf-8-validate": "^5.0.2",
     "warframe-name-generator": "^1.0.2",
     "zlib-sync": "^0.1.6"
   },
   "devDependencies": {
-    "eslint": "^6.1.0",
-    "eslint-config-airbnb-base": "^14.0.0",
-    "eslint-plugin-import": "^2.18.2"
+    "eslint": "^6.8.0",
+    "eslint-config-airbnb-base": "^14.1.0",
+    "eslint-plugin-import": "^2.20.2"
   },
   "license": "Apache-2.0",
   "engines": {

src/CommonFunctions.js

@@ -15,7 +15,7 @@ const welcomes = require('./resources/welcomes.json');
 
 const {
   eventTypes, rewardTypes, opts, fissures, syndicates, twitter, conclave, deals, clantech,
-  resources, nightwave,
+  resources, nightwave, twitch,
 } = require('./resources/trackables.json');
 
 const rssFeeds = require('./resources/rssFeeds');
@@ -102,6 +102,7 @@ const trackableEvents = {
   rss: rssFeeds.map(feed => feed.key),
   arbitration: [],
   kuva: [],
+  twitch,
   opts,
 };
 
@@ -110,7 +111,7 @@ trackableEvents.events.push(...trackableEvents.rss);
 const tTemp = [];
 twitter.types.forEach((type) => {
   twitter.accounts.forEach((account) => {
-    const id = `twitter.${type}.${account}`;
+    const id = `twitter.${account}.${type}`;
     if (!trackableEvents[`twitter.${type}`]) {
       trackableEvents[`twitter.${type}`] = [];
     }
@@ -125,26 +126,28 @@ const arbiTemp = [];
 const kuvaTemp = [];
 Object.keys(missionTypes).forEach((type) => {
   // These will be re-enabled when arbitrations/kuva are ready
-  if (missionTypes[type]) {
+  if (missionTypes[type].arbi) {
     factions.forEach((faction) => {
       arbiTemp.push(`arbitration.${faction}.${type}`);
     });
   }
   kuvaTemp.push(`kuva.${type}`);
 
-  // Construct Fissure types
-  fissures.tiers.forEach((tier) => {
-    const id = `fissures.${tier}.${type}`;
-    if (!trackableEvents[`fissures.${tier}`]) {
-      trackableEvents[`fissures.${tier}`] = [];
-    }
-    trackableEvents[`fissures.${tier}`].push(id);
-    if (!trackableEvents[`fissures.${type}`]) {
-      trackableEvents[`fissures.${type}`] = [];
-    }
-    trackableEvents[`fissures.${type}`].push(id);
-    fTemp.push(id);
-  });
+  if (missionTypes[type].fissure) {
+    // Construct Fissure types
+    fissures.tiers.forEach((tier) => {
+      const id = `fissures.${tier}.${type}`;
+      if (!trackableEvents[`fissures.${tier}`]) {
+        trackableEvents[`fissures.${tier}`] = [];
+      }
+      trackableEvents[`fissures.${tier}`].push(id);
+      if (!trackableEvents[`fissures.${type}`]) {
+        trackableEvents[`fissures.${type}`] = [];
+      }
+      trackableEvents[`fissures.${type}`].push(id);
+      fTemp.push(id);
+    });
+  }
 });
 // gotta make sure this is outside the loop
 // and after it completes so all the generated ones are first
@@ -157,6 +160,7 @@ trackableEvents.events.push(
   ...trackableEvents.fissures,
   ...trackableEvents.arbitration,
   ...trackableEvents.kuva,
+  ...trackableEvents.twitch,
 );
 
 const dyn = [
@@ -781,7 +785,7 @@ const getChannel = (channelsParam, message, channels) => {
   let { channel } = message;
   let channelsColl;
   if (message.guild) {
-    channelsColl = message.guild.channels;
+    channelsColl = message.guild.channels.cache;
   } else {
     channelsColl = new Collection();
     channelsColl.set(message.channel.id, message.channel);

src/EventHandler.js

@@ -43,13 +43,14 @@ class EventHandler {
     });
 
     if (this.handlers.length !== 0) {
-      this.logger.debug('Decaching handles');
+      this.logger.silly('Decaching handles');
       files.forEach((f) => {
         decache(path.join(handlersDir, f));
       });
     }
 
-    this.logger.debug(`Loading handles: ${files}`);
+    this.logger.info('Loading handles');
+    this.logger.debug(`${files}`);
 
     this.handlers = files.map((f) => {
       try {
@@ -58,7 +59,7 @@ class EventHandler {
         if (Handler.prototype instanceof BaseEventHandler) {
           const handler = new Handler(this.bot);
 
-          this.logger.debug(`Adding ${handler.id}`);
+          this.logger.silly(`Adding ${handler.id}`);
           return handler;
         }
         return null;

src/Logger.js

@@ -6,6 +6,17 @@ require('colors');
 
 Sentry.init(process.env.RAVEN_URL, { autoBreadcrumbs: true });
 
+const { WebhookClient } = require('discord.js');
+
+const scope = (process.env.SCOPE || 'worker').toUpperCase();
+
+const ErrorEmbed = require('./embeds/ErrorEmbed');
+
+let errorHook;
+if (process.env.CONTROL_WH_ID) {
+  errorHook = new WebhookClient(process.env.CONTROL_WH_ID, process.env.CONTROL_WH_TOKEN);
+}
+
 /**
  * A collection of methods for logging
  * @property {function} silly   - silly level of debugging
@@ -25,23 +36,23 @@ const l = {
 };
 const levels = {
   SILLY: 'grey',
-  DEBUG: 'cyan',
+  DEBUG: 'brightYellow',
   INFO: 'blue',
   WARN: 'orange',
   ERROR: 'red',
   FATAL: 'magenta',
 };
 const scopes = {
   BOT: 'yellow',
-  WORKER: 'green',
+  WORKER: 'grey',
 };
 
 const colorify = (level, map) => level[map[level] || 'red'];
-const fmt = (level, scope, msg) => `[${colorify(scope, scopes)}] ${(colorify(level, levels) || 'ukn').toLowerCase()}: ${msg}`;
+const fmt = (level, msg) => `[${colorify(scope, scopes)}] ${(colorify(level, levels) || 'ukn').toLowerCase()}: ${msg}`;
 
 Object.keys(levels).forEach((level) => {
   Logger.prototype[level.toLowerCase()] = (message) => {
-    const simple = fmt(level, process.env.SCOPE || 'BOT', message);
+    const simple = fmt(level, message);
     const isActive = Object.keys(levels).indexOf(level) >= Object.keys(levels).indexOf(l.logLevel);
     const nonError = Object.keys(levels).indexOf(level) < Object.keys(levels).indexOf('ERROR');
     if (isActive && nonError) {
@@ -54,13 +65,14 @@ Object.keys(levels).forEach((level) => {
       process.exit(4);
     }
     if (level.toLowerCase() === 'error') {
-      console.error(simple);
-      if (typeof message === 'object') {
-        console.error(message);
-      }
       if (Sentry) {
         Sentry.captureException(message);
       }
+      if (errorHook) {
+        errorHook.send(new ErrorEmbed(message));
+      } else {
+        console.error(simple);
+      }
     }
   };
 });

src/WorldStateCache.js

@@ -3,6 +3,7 @@
 const EventEmitter = require('events');
 const { apiBase } = require('./CommonFunctions.js');
 const fetch = require('./resources/Fetcher');
+const logger = require('./Logger');
 
 const worldStateURLs = {
   pc: `${apiBase}/pc`,
@@ -12,7 +13,7 @@ const worldStateURLs = {
 };
 
 class WorldStateCache extends EventEmitter {
-  constructor(platform, timeout, logger = console) {
+  constructor(platform, timeout) {
     super();
     this.url = worldStateURLs[platform];
     this.timeout = timeout;

src/bot.js

@@ -2,14 +2,12 @@
 
 const { Client, WebhookClient } = require('discord.js');
 
-const WorldStateCache = require('./WorldStateCache');
 const WorldStateClient = require('./resources/WorldStateClient');
 const CommandManager = require('./CommandManager');
 const EventHandler = require('./EventHandler');
 // const Tracker = require('./Tracker');
 
 const MessageManager = require('./settings/MessageManager');
-const Notifier = require('./notifications/Notifier');
 const Database = require('./settings/Database');
 const logger = require('./Logger');
 
@@ -128,18 +126,6 @@ class Genesis {
      */
     this.settings = new Database(this);
 
-    /**
-     * Objects holding worldState data, one for each platform
-     * @type {Object.<WorldStateCache>}
-     */
-    this.worldStates = {};
-
-    const worldStateTimeout = process.env.WORLDSTATE_TIMEOUT || 60000;
-    ['pc', 'ps4', 'xb1', 'swi']
-      .forEach((platform) => {
-        this.worldStates[platform] = new WorldStateCache(platform, worldStateTimeout, this.logger);
-      });
-
     this.ws = new WorldStateClient(this.logger);
     this.messageManager = new MessageManager(this);
 
@@ -162,7 +148,6 @@ class Genesis {
     this.shards = shards;
     this.shardTotal = process.env.SHARDS || 1;
     this.clusterId = process.env.CLUSTER_ID || 0;
-    this.notifier = new Notifier(this);
     // this.tracker = new Tracker(this);
 
     if (process.env.CONTROL_WH_ID) {
@@ -218,7 +203,6 @@ class Genesis {
     try {
       await this.client.login(this.token);
       this.logger.debug('Logged in with token.');
-      await this.notifier.start();
     } catch (err) {
       const type = ((err && err.toString()) || '').replace(/Error \[(.*)\]: .*/ig, '$1');
       if (!unlog.includes(type)) {

src/commands/Builds/AddBuild.js

@@ -18,7 +18,7 @@ class AddBuild extends Command {
    */
   constructor(bot) {
     super(bot, 'builds.add', 'add build', 'Create a temporary room.', 'UTIL');
-    this.regex = new RegExp(`^(?:${this.call}|ab)\\s?(.+)?`, 'i');
+    this.regex = new RegExp(`^(?:${this.call}|ab)\\s(.+)?`, 'i');
 
     this.usages = [
       { description: 'Display instructions for creating a new build with Genesis', parameters: [] },

src/commands/Ondemand/Pricecheck.js

@@ -35,6 +35,11 @@ class PriceCheck extends Command {
   async run(message, ctx) {
     try {
       const item = message.strippedContent.match(this.regex)[1];
+      if (item.length > 16 || item.length < 2) {
+        await message.reply('Search has a 16 character limit.');
+        return this.messageManager.statuses.FAILURE;
+      }
+
       const sentMessage = await message.channel.send('', { embed: inProgressEmbed });
 
       const result = await this.ws.pricecheck(item, { platform: ctx.platform });

src/commands/Ondemand/WhereIs.js

@@ -43,18 +43,50 @@ class Whereis extends Command {
     try {
       query = query.trim().toLowerCase();
       const queryWReplaces = query.replace(/prime/ig, 'p.').replace(/blueprint/ig, 'bp');
-      let results = await this.ws.search('drops', queryWReplaces);
+      const queryResults = (await this.ws.search('drops', queryWReplaces))
+        .map((result) => {
+          const r = {
+            item: result.item,
+            rarity: result.rarity,
+            chance: `${String(parseFloat(result.chance).toFixed(2)).padEnd(5, '0')}%`,
+            chanceNum: parseFloat(result.chance).toFixed(2),
+            place: result.place
+              .replace('Level ', '')
+              .replace(' Orb Vallis Bounty', '')
+              .replace(' Cetus Bounty', '')
+              .trim(),
+          };
+          r.place = r.place.split('/')[1] || r.place;
+          return r;
+        });
 
-      results = results.map(result => ({
-        item: result.item,
-        rarity: result.rarity,
-        chance: `${String(parseFloat(result.chance).toFixed(2)).padEnd(5, '0')}%`,
-        place: result.place
-          .replace('Level ', '')
-          .replace('Orb Vallis Bounty', 'Bounty')
-          .replace('Cetus Bounty', 'Bounty')
-          .trim(),
-      }));
+      let results = [];
+
+      const map = new Map();
+      for (const item of queryResults) {
+        const isRelic = item.place.includes('Relic');
+        const relic = item.place.split('(')[0].trim();
+        if (isRelic && (!map.has(relic) || map.get(relic) < item.chanceNum)) {
+          if (map.has(relic)) {
+            let indexToRemove;
+            results.forEach((urelic, index) => {
+              if (urelic.place.includes(relic)) {
+                indexToRemove = index;
+              }
+            });
+            if (typeof indexToRemove !== 'undefined') {
+              results.splice(indexToRemove, 1);
+            }
+          }
+          map.set(relic, item.chanceNum);
+          results.push(item);
+        } else if (!isRelic && (!map.has(item.place) || map.get(item.place) < item.chanceNum)) {
+          map.set(item.place, item.chanceNum);
+          results.push(item);
+        }
+      }
+
+      results = [...(new Set(results))];
 
       const longestName = results.length ? results.map(result => result.item)
         .reduce((a, b) => (a.length > b.length ? a : b)) : '';