Adjust cache warming schedule; add back crawler

index.js

@@ -68,10 +68,16 @@ const updateGalnetNews = require('./lib/cron-tasks/galnet-news')
   if (process?.env?.NODE_ENV === 'development') {
     console.log('Cache warming disabled')
   } else {
-    // Ensure this happens at startup without forcing the server to wait for it
     console.log('Cache warming enabled')
-    cron.schedule('0 */5 * * * *', () => warmCache())
+    // Ensure this happens at startup without forcing the server to wait for it
     warmCache()
+    // Schedule cache warming to re-run every couple of minutes. This results in
+    // performance very similar to a pure RAM disk most of the time (because
+    // the data _is_ in RAM) but withotu the complexity of having to deal with
+    // actually syncing an in-memory database to disk, because the OS will
+    // handle it. A physical disk parition for this data in a RAID 1 array with
+    // a RAM drive would be ideal, but this is lower effort on my part.
+    cron.schedule('0 */2 * * * *', () => warmCache())
   }
 
   console.log('Ardent API service started!')

lib/crawler.js

@@ -0,0 +1,23 @@
+// This script was originally written to force cache warming, but with that
+// code having been refactored am now using it for performance testing.
+const dbAsync = require('./db/db-async')
+const { ARDENT_API_HOSTNAME } = require('./consts')
+
+module.exports = async (debug = true) => {
+  console.time('Crawl commodities')
+  try {
+    const commodities = await dbAsync.all('SELECT DISTINCT(commodityName) FROM commodities')
+    if (debug === true) console.log(`Warming cache for ${ARDENT_API_HOSTNAME}`)
+    for (let i = 0; i < commodities.length; i++) {
+      const { commodityName } = commodities[i]
+      const url = `https://${ARDENT_API_HOSTNAME}/v1/commodity/name/${commodityName}/imports`
+      if (debug === true) console.time(`${i+1} of ${commodities.length} ${commodityName}`)
+      const res = await fetch(url)
+      if (!res.ok) console.error(`Crawler error fetching: ${url}`)
+      if (debug === true) console.timeEnd(`${i+1} of ${commodities.length} ${commodityName}`)
+    }
+  } catch (e) {
+    return console.error('Crawler failed:', e)
+  }
+  console.timeEnd('Crawl commodities')
+}

package.json

@@ -1,11 +1,12 @@
 {
   "name": "ardent-api",
-  "version": "4.11.1",
+  "version": "4.11.2",
   "description": "Ardent API provides access to data submitted to EDDN",
   "main": "index.js",
   "scripts": {
     "start": "node index.js",
     "warm-cache": "node scripts/warm-cache.js",
+    "crawler": "node scripts/crawler.js",
     "dev": "cross-env NODE_ENV=development node index.js",
     "lint": "npx standard --fix"
   },

scripts/crawler.js

@@ -0,0 +1,5 @@
+const crawler = require('../lib/crawler')
+;(async () => {
+  await crawler()
+  process.exit()
+})()