feat: cache all remote data for quicker startup

.gitignore

@@ -84,3 +84,7 @@ src/**/*.custom.css
 server/src/models/queries/*
 /logs/*
 !/logs/.gitkeep
+
+# Cache
+server/.cache/*
+!/server/.cache/.gitkeep

packages/logger/lib/index.js

@@ -14,6 +14,7 @@ const HELPERS = /** @type {const} */ ({
   info: chalk.blue('ℹ'),
   warn: chalk.yellow('⚠'),
   error: chalk.red('✖'),
+
   telegram: chalk.hex('#26A8EA')('[TELEGRAM]'),
   discord: chalk.hex('#7289da')('[DISCORD]'),
   cache: chalk.blueBright('[CACHE]'),

server/src/index.js

@@ -29,7 +29,7 @@ const Clients = require('./services/Clients')
 const sessionStore = require('./services/sessionStore')
 const rootRouter = require('./routes/rootRouter')
 const pkg = require('../../package.json')
-const getAreas = require('./services/areas')
+const { loadLatestAreas } = require('./services/areas')
 const { connection } = require('./db/knexfile.cjs')
 const startApollo = require('./graphql/server')
 require('./services/watcher')
@@ -302,8 +302,9 @@ startApollo(httpServer).then((server) => {
 connection.migrate
   .latest()
   .then(() => connection.destroy())
+  .then(() => Db.getDbContext())
   .then(async () => {
-    await Db.getDbContext()
+    httpServer.listen(config.getSafe('port'), config.getSafe('interface'))
     await Promise.all([
       Db.historicalRarity(),
       Db.getFilterContext(),
@@ -318,9 +319,8 @@ connection.migrate
       Event.getMasterfile(Db.historical, Db.rarity),
       Event.getInvasions(config.getSafe('api.pogoApiEndpoints.invasions')),
       Event.getWebhooks(),
-      getAreas().then((res) => (config.areas = res)),
+      loadLatestAreas().then((res) => (config.areas = res)),
     ])
-    httpServer.listen(config.getSafe('port'), config.getSafe('interface'))
     const text = rainbow(
       `ℹ ${new Date()
         .toISOString()

server/src/routes/api/v1/area.js

@@ -3,14 +3,14 @@ const router = require('express').Router()
 
 const config = require('@rm/config')
 const { log, HELPERS } = require('@rm/logger')
-const getAreas = require('../../../services/areas')
+const { loadLatestAreas } = require('../../../services/areas')
 
 const reactMapSecret = config.getSafe('api.reactMapSecret')
 
 router.get('/reload', async (req, res) => {
   try {
     if (reactMapSecret && req.headers['react-map-secret'] === reactMapSecret) {
-      const newAreas = await getAreas()
+      const newAreas = await loadLatestAreas()
       config.areas = newAreas
 
       res.status(200).json({ status: 'ok', message: 'reloaded areas' })

server/src/services/DbCheck.js

@@ -6,6 +6,7 @@ const config = require('@rm/config')
 
 const { log, HELPERS } = require('@rm/logger')
 const { getBboxFromCenter } = require('./functions/getBbox')
+const { setCache, getCache } = require('./cache')
 
 const softLimit = config.getSafe('api.searchSoftKmLimit')
 const hardLimit = config.getSafe('api.searchHardKmLimit')
@@ -37,13 +38,13 @@ module.exports = class DbCheck {
     this.searchLimit = config.getSafe('api.searchResultsLimit')
     this.rarityPercents = config.getSafe('rarity.percents')
     this.models = {}
-    this.questConditions = {}
     this.endpoints = {}
-    this.rarity = {}
-    this.historical = {}
-    this.filterContext = {
+    this.questConditions = getCache('questConditions.json', {})
+    this.rarity = getCache('rarity.json', {})
+    this.historical = getCache('historical.json', {})
+    this.filterContext = getCache('filterContext.json', {
       Route: { maxDistance: 0, maxDuration: 0 },
-    }
+    })
     this.reactMapDb = null
     this.connections = config
       .getSafe('database.schemas')
@@ -246,7 +247,7 @@ module.exports = class DbCheck {
    * @param {boolean} historical
    * @returns {void}
    */
-  setRarity(results, historical = false) {
+  async setRarity(results, historical = false) {
     const base = {}
     const mapKey = historical ? 'historical' : 'rarity'
     let total = 0
@@ -277,6 +278,7 @@ module.exports = class DbCheck {
         this[mapKey][id] = 'common'
       }
     })
+    await setCache(`${mapKey}.json`, this[mapKey])
   }
 
   async historicalRarity() {
@@ -292,7 +294,7 @@ module.exports = class DbCheck {
                 .groupBy('pokemon_id'),
         ),
       )
-      this.setRarity(
+      await this.setRarity(
         results.map((result) =>
           Object.fromEntries(
             result.map((pkmn) => [`${pkmn.pokemon_id}`, +pkmn.total]),
@@ -592,9 +594,10 @@ module.exports = class DbCheck {
               Object.values(titles),
             ]),
           )
+          await setCache('questConditions.json', this.questConditions)
         }
         if (model === 'Pokemon') {
-          this.setRarity(results, false)
+          await this.setRarity(results, false)
         }
         if (results.length === 1) return results[0].available
         if (results.length > 1) {
@@ -624,19 +627,28 @@ module.exports = class DbCheck {
    * Builds filter context for all models
    */
   async getFilterContext() {
-    if (this.models.Route) {
-      const results = await Promise.all(
-        this.models.Route.map(({ SubModel, ...source }) =>
-          SubModel.getFilterContext(source),
-        ),
-      )
-      this.filterContext.Route.maxDistance = Math.max(
-        ...results.map((result) => result.max_distance),
-      )
-      this.filterContext.Route.maxDuration = Math.max(
-        ...results.map((result) => result.max_duration),
+    try {
+      if (this.models.Route) {
+        const results = await Promise.all(
+          this.models.Route.map(({ SubModel, ...source }) =>
+            SubModel.getFilterContext(source),
+          ),
+        )
+        this.filterContext.Route.maxDistance = Math.max(
+          ...results.map((result) => result.max_distance),
+        )
+        this.filterContext.Route.maxDuration = Math.max(
+          ...results.map((result) => result.max_duration),
+        )
+        log.info(HELPERS.db, 'Updating filter context for routes')
+        await setCache('filterContext.json', this.filterContext)
+      }
+    } catch (e) {
+      log.error(
+        HELPERS.db,
+        'If you are using RDM, you likely do not have a routes table. Remove `route` from the `useFor` array in your config',
+        e,
       )
-      log.info(HELPERS.db, 'Updating filter context for routes')
     }
   }
 }

server/src/services/EventManager.js

@@ -9,6 +9,7 @@ const { log, HELPERS } = require('@rm/logger')
 const { generate, read } = require('@rm/masterfile')
 
 const PoracleAPI = require('./api/Poracle')
+const { getCache, setCache } = require('./cache')
 
 class EventManager {
   constructor() {
@@ -19,21 +20,33 @@ class EventManager {
       'invasions' in this.masterfile ? this.masterfile.invasions : {}
 
     /** @type {{[key in keyof import('@rm/types').Available]: string[] }} */
-    this.available = {
+    this.available = getCache('available.json', {
       gyms: [],
       pokestops: [],
       pokemon: [],
       nests: [],
-    }
-    this.uicons = []
-    this.uaudio = []
+    })
+    this.uicons = getCache('uicons.json', [])
+    this.uaudio = getCache('uaudio.json', [])
     this.uiconsBackup = {}
     this.uaudioBackup = {}
     this.baseUrl =
       'https://raw.githubusercontent.com/WatWowMap/wwm-uicons-webp/main'
 
     /** @type {Record<string, InstanceType<typeof PoracleAPI>>} */
-    this.webhookObj = {}
+    this.webhookObj = Object.fromEntries(
+      config
+        .getSafe('webhooks')
+        .filter((x) => x.enabled)
+        .map((webhook) => {
+          const api = new PoracleAPI(webhook)
+          if (api.initFromCache()) {
+            return [api.name, api]
+          }
+          return [api.name, null]
+        })
+        .filter(([, api]) => api),
+    )
     /** @type {import('./Clients').ClientObject} */
     this.Clients = {}
   }
@@ -91,6 +104,7 @@ class EventManager {
 
       return 0
     })
+    await setCache('available.json', this.available)
   }
 
   /**
@@ -295,6 +309,7 @@ class EventManager {
       }
     }
     this[type] = Object.values(this[`${type}Backup`])
+    await setCache(`${type}.json`, this[type])
   }
 
   /**
@@ -361,18 +376,22 @@ class EventManager {
   }
 
   async getWebhooks() {
-    await Promise.all(
+    const apis = await Promise.allSettled(
       config
         .getSafe('webhooks')
         .filter((x) => x.enabled)
         .map(async (webhook) => {
           const api = new PoracleAPI(webhook)
           await api.init()
-          Object.assign(this.webhookObj, {
-            [webhook.name]: api,
-          })
+          return api
         }),
     )
+    for (let i = 0; i < apis.length; i += 1) {
+      const item = apis[i]
+      if (item.status === 'fulfilled' && item.value) {
+        this.webhookObj[item.value.name] = item.value
+      }
+    }
   }
 }
 

server/src/services/api/Poracle.js

@@ -1,5 +1,6 @@
 const { log, HELPERS } = require('@rm/logger')
 const fetchJson = require('./fetchJson')
+const { setCache, getCache } = require('../cache')
 
 const PLATFORMS = /** @type {const} */ (['discord', 'telegram'])
 
@@ -180,6 +181,15 @@ class PoracleAPI {
     }
   }
 
+  initFromCache() {
+    const cached = getCache(`${this.name}-webhook.json`)
+    if (!cached) {
+      log.warn(HELPERS.webhooks, `${this.name} webhook not found in cache`)
+      return false
+    }
+    Object.assign(this, cached)
+  }
+
   async init() {
     try {
       await Promise.all([
@@ -190,6 +200,7 @@ class PoracleAPI {
       this.ui = this.generateUi()
       this.lastFetched = Date.now()
       log.info(HELPERS.webhooks, `${this.name} webhook initialized`)
+      await setCache(`${this.name}-webhook.json`, this)
     } catch (e) {
       log.error(HELPERS.webhooks, `Error initializing ${this.name} webhook`, e)
     }