diff --git a/index.js b/index.js index c41b98b..293c046 100644 --- a/index.js +++ b/index.js @@ -55,18 +55,6 @@ const updateGalnetNews = require('./lib/cron-tasks/galnet-news') router.get('/api', (ctx) => { ctx.body = printStats() }) app.use(router.routes()) - // Run task to warm up the cache every 15 minutes - if (process?.env?.NODE_ENV === 'development') { - console.log('Cache warming disabled') - } else { - // Experimented with disabling cache warming after the system upgrade, but - // it still makes an appreciable difference to request times and keeps - // request times under 1 second, so leaving it on. It takes a bit under - // 3 minutes to complete, running every 15 minutes seems frequent enough. - console.log('Cache warming enabled') - cron.schedule('0 */15 * * * *', () => warmCache()) - } - updateCommodityTicker() cron.schedule('0 */5 * * * *', async () => updateCommodityTicker()) @@ -75,6 +63,17 @@ const updateGalnetNews = require('./lib/cron-tasks/galnet-news') app.listen(ARDENT_API_LOCAL_PORT) console.log(printStats()) + + // Schedule task to try to keep the cache warm + if (process?.env?.NODE_ENV === 'development') { + console.log('Cache warming disabled') + } else { + // Ensure this happens at startup without forcing the server to wait for it + console.log('Cache warming enabled') + cron.schedule('0 */5 * * * *', () => warmCache()) + warmCache() + } + console.log('Ardent API service started!') })() diff --git a/lib/warm-cache.js b/lib/warm-cache.js index b76f50e..7df551d 100644 --- a/lib/warm-cache.js +++ b/lib/warm-cache.js @@ -1,39 +1,20 @@ -const dbAsync = require('./db/db-async') -const { ARDENT_API_HOSTNAME } = require('./consts') +const { exec } = require('child_process') +const commandExistsSync = require('command-exists').sync +const { ARDENT_TRADE_DB } = require('./consts') -// Warms the cache by hitting the 'import' stats endpoint for every commodity. -// This has the knock-on effect of ensuring that all the commodity data is in -// memory and so getting export data for commodities is also equally fast. -// -// This script was created because low traffic volumes seems to lead to data in -// SQLite being unloaded from memory, resulting in queries taking 10-20 seconds -// instead of being sub-second. -// -// I was unable to resolve the issue through changes to SQLite cache behaviour. -// The database for star systems is much larger (over 100 million entries) but -// does not have any performance problems, I suspect the much greater volume of -// writes to the trade database and/or RAM constraints on the production server -// are underlying factors triggering the performance issue for commodities. -// -// Queries are performed sequentially to avoid unnecessary load on the server. -// -// This task takes ~15 minutes to run when the cache is cold and ~3 minutes when -// the cache warmed up - the goal is to keep it always warm. -module.exports = async ({ debug = false }) => { - if (debug === true) console.time('Time warm cache') - try { - const commodities = await dbAsync.all('SELECT DISTINCT(commodityName) FROM commodities') - if (debug === true) console.log(`Warming cache for ${ARDENT_API_HOSTNAME}`) - for (let i = 0; i < commodities.length; i++) { - const { commodityName } = commodities[i] - const url = `https://${ARDENT_API_HOSTNAME}/v1/commodity/name/${commodityName}/imports` - if (debug === true) console.time(`${i+1} of ${commodities.length} ${commodityName}`) - const res = await fetch(url) - if (!res.ok) console.error(`Cache warm error fetching: ${url}`) - if (debug === true) console.timeEnd(`${i+1} of ${commodities.length} ${commodityName}`) - } - } catch (e) { - return console.error('Cache warm failed:', e) +module.exports = () => { + if (commandExistsSync('vmtouch')) { + // Try (but don't require) to keep all trade database files in memory cache. + // + // Other databases like the Station and even much larger Systems database + // work fine without being in memory, the trade database is a special case, + // due to the nature of the data and the many ways it can be queried. + // + // Note: Not using vmtouch in daemon mode by design - too many side effects + // but best effort prompting every 5 minutes is fine. It takes between ~90 + // seconds to run from cold boot and < 1 second if already fully cached. + exec(`vmtouch -t ${ARDENT_TRADE_DB}* -m 24G`, (error, stdout, stderr) => { + if (error) console.error(error) + }) } - if (debug === true) console.timeEnd('Time warm cache') } diff --git a/package-lock.json b/package-lock.json index a979a7a..4edb95a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,15 +1,16 @@ { "name": "ardent-api", - "version": "4.10.0", + "version": "4.11.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ardent-api", - "version": "4.10.0", + "version": "4.11.0", "license": "AGPL-3.0", "dependencies": { "better-sqlite3": "^8.3.0", + "command-exists": "^1.2.9", "cross-env": "^7.0.3", "dotenv": "^16.0.3", "koa": "^2.14.2", @@ -662,6 +663,11 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true }, + "node_modules/command-exists": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/command-exists/-/command-exists-1.2.9.tgz", + "integrity": "sha512-LTQ/SGc+s0Xc0Fu5WaKnR0YiygZkm9eKFvyS+fRsU7/ZWFF8ykFM6Pc9aCVf1+xasOOZpO3BAVgVrKvsqKHV7w==" + }, "node_modules/compressible": { "version": "2.0.18", "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", diff --git a/package.json b/package.json index 1d91d02..c4c8b6c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ardent-api", - "version": "4.10.1", + "version": "4.11.0", "description": "Ardent API provides access to data submitted to EDDN", "main": "index.js", "scripts": { @@ -21,6 +21,7 @@ "homepage": "https://github.com/iaincollins/ardent-api#readme", "dependencies": { "better-sqlite3": "^8.3.0", + "command-exists": "^1.2.9", "cross-env": "^7.0.3", "dotenv": "^16.0.3", "koa": "^2.14.2",