Refactor cache warming

index.js

@@ -55,18 +55,6 @@ const updateGalnetNews = require('./lib/cron-tasks/galnet-news')
   router.get('/api', (ctx) => { ctx.body = printStats() })
   app.use(router.routes())
 
-  // Run task to warm up the cache every 15 minutes
-  if (process?.env?.NODE_ENV === 'development') {
-    console.log('Cache warming disabled')
-  } else {
-    // Experimented with disabling cache warming after the system upgrade, but
-    // it still makes an appreciable difference to request times and keeps
-    // request times under 1 second, so leaving it on. It takes a bit under
-    // 3 minutes to complete, running every 15 minutes seems frequent enough.
-    console.log('Cache warming enabled')
-    cron.schedule('0 */15 * * * *', () => warmCache())
-  }
-
   updateCommodityTicker()
   cron.schedule('0 */5 * * * *', async () => updateCommodityTicker())
 
@@ -75,6 +63,17 @@ const updateGalnetNews = require('./lib/cron-tasks/galnet-news')
 
   app.listen(ARDENT_API_LOCAL_PORT)
   console.log(printStats())
+
+  // Schedule task to try to keep the cache warm
+  if (process?.env?.NODE_ENV === 'development') {
+    console.log('Cache warming disabled')
+  } else {
+    // Ensure this happens at startup without forcing the server to wait for it
+    console.log('Cache warming enabled')
+    cron.schedule('0 */5 * * * *', () => warmCache())
+    warmCache()
+  }
+
   console.log('Ardent API service started!')
 })()
 

lib/warm-cache.js

@@ -1,39 +1,20 @@
-const dbAsync = require('./db/db-async')
-const { ARDENT_API_HOSTNAME } = require('./consts')
+const { exec } = require('child_process')
+const commandExistsSync = require('command-exists').sync
+const { ARDENT_TRADE_DB } = require('./consts')
 
-// Warms the cache by hitting the 'import' stats endpoint for every commodity.
-// This has the knock-on effect of ensuring that all the commodity data is in
-// memory and so getting export data for commodities is also equally fast.
-//
-// This script was created because low traffic volumes seems to lead to data in
-// SQLite being unloaded from memory, resulting in queries taking 10-20 seconds
-// instead of being sub-second.
-//
-// I was unable to resolve the issue through changes to SQLite cache behaviour.
-// The database for star systems is much larger (over 100 million entries) but
-// does not have any performance problems, I suspect the much greater volume of
-// writes to the trade database and/or RAM constraints on the production server
-// are underlying factors triggering the performance issue for commodities.
-//
-// Queries are performed sequentially to avoid unnecessary load on the server.
-//
-// This task takes ~15 minutes to run when the cache is cold and ~3 minutes when
-// the cache warmed up - the goal is to keep it always warm.
-module.exports = async ({ debug = false }) => {
-  if (debug === true) console.time('Time warm cache')
-  try {
-    const commodities = await dbAsync.all('SELECT DISTINCT(commodityName) FROM commodities')
-    if (debug === true) console.log(`Warming cache for ${ARDENT_API_HOSTNAME}`)
-    for (let i = 0; i < commodities.length; i++) {
-      const { commodityName } = commodities[i]
-      const url = `https://${ARDENT_API_HOSTNAME}/v1/commodity/name/${commodityName}/imports`
-      if (debug === true) console.time(`${i+1} of ${commodities.length} ${commodityName}`)
-      const res = await fetch(url)
-      if (!res.ok) console.error(`Cache warm error fetching: ${url}`)
-      if (debug === true) console.timeEnd(`${i+1} of ${commodities.length} ${commodityName}`)
-    }
-  } catch (e) {
-    return console.error('Cache warm failed:', e)
+module.exports = () => {
+  if (commandExistsSync('vmtouch')) {
+    // Try (but don't require) to keep all trade database files in memory cache.
+    //
+    // Other databases like the Station and even much larger Systems database
+    // work fine without being in memory, the trade database is a special case,
+    // due to the nature of the data and the many ways it can be queried.
+    //
+    // Note: Not using vmtouch in daemon mode by design - too many side effects
+    // but best effort prompting every 5 minutes is fine. It takes between ~90
+    // seconds to run from cold boot and < 1 second if already fully cached.
+    exec(`vmtouch -t ${ARDENT_TRADE_DB}* -m 24G`, (error, stdout, stderr) => {
+      if (error) console.error(error)
+    })
   }
-  if (debug === true) console.timeEnd('Time warm cache')
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ardent-api",
-  "version": "4.10.1",
+  "version": "4.11.0",
   "description": "Ardent API provides access to data submitted to EDDN",
   "main": "index.js",
   "scripts": {
@@ -21,6 +21,7 @@
   "homepage": "https://github.com/iaincollins/ardent-api#readme",
   "dependencies": {
     "better-sqlite3": "^8.3.0",
+    "command-exists": "^1.2.9",
     "cross-env": "^7.0.3",
     "dotenv": "^16.0.3",
     "koa": "^2.14.2",