Add news feeds

* New galnet news feed
* Refactored commotiy ticker (now leverages cache)

index.js

@@ -20,6 +20,8 @@ const cron = require('node-cron')
 console.log('Loading libraries …')
 const router = require('./router')
 const warmCache = require('./lib/warm-cache')
+const updateCommodityTicker = require('./lib/cron-tasks/commodity-ticker')
+const updateGalnetNews = require('./lib/cron-tasks/galnet-news')
 
 ;(async () => {
   // Start web service
@@ -59,6 +61,12 @@ const warmCache = require('./lib/warm-cache')
     cron.schedule('0 */15 * * * *', () => warmCache())
   }
 
+  updateCommodityTicker()
+  cron.schedule('0 */5 * * * *', async () => updateCommodityTicker())
+
+  updateGalnetNews()
+  cron.schedule('0 */20 * * * *', async () => updateGalnetNews())
+
   app.listen(ARDENT_API_LOCAL_PORT)
   console.log(printStats())
   console.log('Ardent API service started!')

lib/consts.js

@@ -19,6 +19,8 @@ const ARDENT_DATA_DIR = process.env?.ARDENT_DATA_DIR ?? path.join(__dirname, '..
 const ARDENT_CACHE_DIR = process.env?.ARDENT_CACHE_DIR ?? path.join(ARDENT_DATA_DIR, 'cache')
 const ARDENT_BACKUP_DIR = process.env?.ARDENT_BACKUP_DIR ?? path.join(__dirname, '../../ardent-backup')
 
+if (!fs.existsSync(ARDENT_CACHE_DIR)) fs.mkdirSync(ARDENT_CACHE_DIR, { recursive: true })
+
 // Data in the Systems DB assumes these values and needs rebuilding if changes
 const SYSTEM_GRID_SIZE = 100 // In light years
 const SYSTEM_SECTOR_HASH_LENGTH = 8 // Enough to minimise sector ID collisions
@@ -30,6 +32,9 @@ const ARDENT_TRADE_DB = path.join(ARDENT_DATA_DIR, '/trade.db')
 
 const DEFAULT_MAX_RESULTS_AGE = 90
 
+const ARDENT_MARKET_TICKER_CACHE = `${ARDENT_CACHE_DIR}/commodity-ticker.js`
+const ARDENT_GALNET_NEWS_CACHE = `${ARDENT_CACHE_DIR}/galnet-news.js`
+
 module.exports = {
   ARDENT_API_HOSTNAME,
   ARDENT_API_LOCAL_PORT,
@@ -43,5 +48,7 @@ module.exports = {
   ARDENT_LOCATIONS_DB,
   ARDENT_STATIONS_DB,
   ARDENT_TRADE_DB,
-  DEFAULT_MAX_RESULTS_AGE
+  DEFAULT_MAX_RESULTS_AGE,
+  ARDENT_MARKET_TICKER_CACHE,
+  ARDENT_GALNET_NEWS_CACHE
 }

lib/cron-tasks/commodity-ticker.js

@@ -0,0 +1,49 @@
+const fs = require('fs')
+const { getISOTimestamp } = require('../utils/dates')
+const dbAsync = require('../db/db-async')
+const { ARDENT_MARKET_TICKER_CACHE } = require('../consts')
+
+module.exports = async () => {
+  const stations = await dbAsync.all(`
+    SELECT * FROM stations.stations AS s WHERE
+      s.stationType != 'Fleet Carrier'
+      AND s.stationType IS NOT NULL
+    GROUP BY s.stationName
+    ORDER BY s.updatedAt DESC
+    LIMIT 50`)
+
+    const imports = []
+    for (const station of stations) {
+      const stationImport = await dbAsync.get(`
+      SELECT * FROM trade.commodities as c
+        WHERE c.marketId = @marketId
+        AND (c.demand > 1000 OR c.demand = 0)
+        AND c.demandBracket = 3
+        AND c.updatedAt > '${getISOTimestamp('-1')}'
+      ORDER BY c.sellPrice DESC
+      LIMIT 5`, { marketId: station.marketId })
+      if (stationImport) imports.push(stationImport)
+    }
+
+    const exports = []
+    for (const station of stations) {
+      const stationExport = await dbAsync.get(`
+      SELECT * FROM trade.commodities as c
+        WHERE c.marketId = @marketId
+        AND c.stock > 1000
+        AND c.stockBracket = 3
+        AND c.updatedAt > '${getISOTimestamp('-1')}'
+      ORDER BY c.buyPrice DESC
+      LIMIT 5`, { marketId: station.marketId })
+      if (stationExport) exports.push(stationExport)
+    }
+
+  const data = [
+    ...imports,
+    ...exports
+    ]
+    .sort((a, b) => a.updatedAt.localeCompare(b.updatedAt)) // Sort results by recency
+    .filter((obj1, i, arr) => arr.findIndex(obj2 => (obj2.marketId === obj1.marketId)) === i) // Filter so only one entry for each station
+
+  fs.writeFileSync(ARDENT_MARKET_TICKER_CACHE, JSON.stringify(data, null, 2))
+}

lib/cron-tasks/galnet-news.js

@@ -0,0 +1,24 @@
+const fs = require('fs')
+const { ARDENT_GALNET_NEWS_CACHE } = require('../consts')
+
+const GALNET_NEWS_FEED = 'https://cms.zaonce.net/en-GB/jsonapi/node/galnet_article'
+
+module.exports = async () => {
+  try {
+    const req = await fetch(GALNET_NEWS_FEED)
+    const json = await req.json()
+
+    const data = json?.data.map(item => ({
+      published: new Date(item.attributes.published_at).toISOString(),
+      date: item.attributes['field_galnet_date'],
+      title: item.attributes.title,
+      text: item.attributes.body.value.replace(/\r/g, ''),
+      slug: item.attributes.field_slug,
+      url: `https://community.elitedangerous.com/galnet/uid/${item.attributes.field_galnet_guid}`
+    }))
+
+    fs.writeFileSync(ARDENT_GALNET_NEWS_CACHE, JSON.stringify(data, null, 2))
+  } catch (e) {
+    console.error('Failed to fetch galnet news', e)
+  }
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ardent-api",
-  "version": "4.0.0",
+  "version": "4.1.0",
   "description": "Ardent API provides access to data submitted to EDDN",
   "main": "index.js",
   "scripts": {

router/api/commodities.js

@@ -2,7 +2,7 @@ const fs = require('fs')
 const path = require('path')
 const { paramAsBoolean, paramAsInt } = require('../../lib/utils/parse-query-params')
 const dbAsync = require('../../lib/db/db-async')
-const { ARDENT_CACHE_DIR, DEFAULT_MAX_RESULTS_AGE } = require('../../lib/consts')
+const { ARDENT_CACHE_DIR, DEFAULT_MAX_RESULTS_AGE, ARDENT_MARKET_TICKER_CACHE } = require('../../lib/consts')
 const NotFoundResponse = require('../../lib/response/not-found')
 const { getISOTimestamp } = require('../../lib/utils/dates')
 
@@ -17,7 +17,7 @@ module.exports = (router) => {
     ctx.body = JSON.parse(fs.readFileSync(COMMODITIES_REPORT)).commodities
   })
 
-  // This is an undocumented and unsupproted route likely to change in future.
+  // deprecated - will be removed in a future update
   router.get('/api/v1-beta/commodities/ticker', async (ctx, next) => {
     const stations = await dbAsync.all(`
     SELECT * FROM stations.stations AS s WHERE

router/api/news.js

@@ -0,0 +1,11 @@
+const fs = require('fs')
+const { ARDENT_GALNET_NEWS_CACHE, ARDENT_MARKET_TICKER_CACHE } = require('../../lib/consts')
+
+module.exports = (router) => {
+  router.get('/api/v1/news/galnet', async (ctx, next) => {
+    ctx.body = JSON.parse(fs.readFileSync(ARDENT_GALNET_NEWS_CACHE))
+  })
+  router.get('/api/v1/news/commodities', async (ctx, next) => {
+    ctx.body = JSON.parse(fs.readFileSync(ARDENT_MARKET_TICKER_CACHE))
+  })
+}

router/api/search.js

@@ -3,11 +3,11 @@ const dbAsync = require('../../lib/db/db-async')
 module.exports = (router) => {
   router.get('/api/v1/search/system/name/:systemName', async (ctx, next) => {
     const { systemName } = ctx.params
-    if (systemName.length < 3) {
+    if (systemName.length < 1) {
       ctx.status = 406
       ctx.body = {
         error: 'System name too short',
-        message: 'Searching for a system by name requires at least the first 3 characters of the name'
+        message: 'Searching for a system by name requires at least 1 character'
       }
       return null
     }

router/index.js

@@ -4,11 +4,11 @@ const KoaRouter = require('koa-router')
 const Package = require('../package.json')
 const { ARDENT_API_HOSTNAME, ARDENT_CACHE_DIR, ARDENT_BACKUP_DIR } = require('../lib/consts')
 const routes = {
+  news: require('./api/news'),
   commodities: require('./api/commodities'),
   systems: require('./api/systems'),
   markets: require('./api/markets'),
   search: require('./api/search')
-  // locations: require('./api/locations') // TODO
 }
 const router = new KoaRouter()
 
@@ -42,6 +42,7 @@ router.get('/api/v1/backup', (ctx, next) => {
   }
 })
 
+routes.news(router)
 routes.commodities(router)
 routes.systems(router)
 routes.markets(router)