feat: processings can manage secrets

Dockerfile

@@ -17,6 +17,21 @@ ADD package.json package-lock.json ./
 RUN jq '.version="build"' package.json | sponge package.json
 RUN jq '.version="build"' package-lock.json | sponge package-lock.json
 
+# =============================
+# Tippecanoe builder
+# =============================
+FROM base AS tippecanoe-builder
+
+WORKDIR /tmp/
+RUN apk add --no-cache curl cmake make g++
+RUN apk add git zlib-dev sqlite-dev bash
+RUN git clone https://github.com/mapbox/tippecanoe.git
+WORKDIR /tmp/tippecanoe
+RUN git checkout 1.36.0
+RUN make -j
+RUN make install
+RUN test -f /usr/local/bin/tippecanoe
+
 # =============================
 # Full dependencies installation (for types and building)
 # =============================
@@ -61,17 +76,6 @@ RUN npm -w ui run build
 # =============================
 FROM installer AS worker-installer
 
-# install tippecanoe for some processings
-WORKDIR /tmp/
-RUN apk add --no-cache curl cmake make g++
-RUN apk add git zlib-dev sqlite-dev bash
-RUN git clone https://github.com/mapbox/tippecanoe.git
-WORKDIR /tmp/tippecanoe
-RUN git checkout 1.36.0
-RUN make -j
-RUN make install
-RUN test -f /usr/local/bin/tippecanoe
-
 RUN npm ci -w worker --prefer-offline --omit=dev --omit=optional --omit=peer --no-audit --no-fund && \
     npx clean-modules --yes
 RUN mkdir -p /app/worker/node_modules
@@ -88,6 +92,7 @@ RUN test -f /usr/bin/ogr2ogr
 RUN ln -s /usr/lib/libproj.so.25 /usr/lib/libproj.so
 RUN test -f /usr/lib/libproj.so
 
+COPY --from=tippecanoe-builder /usr/local/bin/tippecanoe /usr/local/bin/tippecanoe
 COPY --from=worker-installer /app/node_modules node_modules
 COPY worker worker
 COPY shared shared
@@ -96,7 +101,6 @@ COPY --from=types /app/worker/config worker/config
 COPY --from=types /app/api/types api/types
 COPY --from=worker-installer /app/worker/node_modules worker/node_modules
 COPY --from=worker-installer /app/shared/node_modules shared/node_modules
-COPY --from=worker-installer /usr/local/bin/tippecanoe /usr/local/bin/tippecanoe
 COPY package.json README.md LICENSE BUILD.json* ./
 
 EXPOSE 9090

api/config/custom-environment-variables.cjs

@@ -1,4 +1,5 @@
 module.exports = {
+  cipherPassword: 'CIPHER_PASSWORD',
   dataDir: 'DATA_DIR',
   tmpDir: 'TMP_DIR',
   defaultLimits: {

api/config/default.cjs

@@ -1,4 +1,5 @@
 module.exports = {
+  cipherPassword: undefined,
   dataDir: '/app/data',
   tmpDir: null, // will be dataDir + '/tmp' if null
   defaultLimits: {

api/config/development.cjs

@@ -1,4 +1,5 @@
 module.exports = {
+  cipherPassword: 'dev',
   dataDir: '../data/development',
   port: 8082,
   privateDirectoryUrl: 'http://localhost:8080',

api/config/test.cjs

@@ -1,4 +1,5 @@
 module.exports = {
+  cipherPassword: 'test',
   dataDir: './data/test',
   port: 8082,
   privateDirectoryUrl: 'http://localhost:8080',

api/config/type/schema.json

@@ -11,6 +11,7 @@
   "title": "Api config",
   "additionalProperties": false,
   "required": [
+    "cipherPassword",
     "dataDir",
     "privateDirectoryUrl",
     "mongoUrl",
@@ -20,6 +21,9 @@
     "secretKeys"
   ],
   "properties": {
+    "cipherPassword": {
+      "type": "string"
+    },
     "dataDir": {
       "type": "string"
     },

api/src/routers/processings.ts

@@ -1,4 +1,5 @@
 import type { Processing } from '#types'
+import type { PrepareFunction } from '@data-fair/lib-common-types/processings.js'
 import type { SessionStateAuthenticated } from '@data-fair/lib-express/index.js'
 
 import Ajv from 'ajv'
@@ -20,6 +21,7 @@ import locks from '#locks'
 import { resolvedSchema as processingSchema } from '#types/processing/index.ts'
 import findUtils from '../utils/find.ts'
 import permissions from '../utils/permissions.ts'
+import { cipher } from '../utils/cipher.ts'
 
 const router = Router()
 export default router
@@ -34,16 +36,30 @@ const sensitiveParts = ['permissions', 'webhookKey', 'config']
  * Check that a processing object is valid
  * Check if the plugin exists
  * Check if the config is valid (only if the processing is activated)
+ * Encrypt secrets if present
  */
 const validateFullProcessing = async (processing: Processing) => {
   (await import('#types/processing/index.ts')).returnValid(processing)
   if (processing.active && !processing.config) throw httpError(400, 'Config is required for an active processing')
   if (!await fs.pathExists(path.join(pluginsDir, processing.plugin))) throw httpError(400, 'Plugin not found')
   if (!processing.config) return // no config to validate
-  const pluginInfo = await fs.readJson(path.join(pluginsDir, path.join(processing.plugin, 'plugin.json')))
+  const pluginInfo = await fs.readJson(path.join(pluginsDir, processing.plugin, 'plugin.json'))
   const configValidate = ajv.compile(pluginInfo.processingConfigSchema)
   const configValid = configValidate(processing.config)
   if (!configValid) throw httpError(400, JSON.stringify(configValidate.errors))
+
+  // Get the plugin file and execute the prepare function if it exists
+  const plugin = await import(path.resolve(process.cwd(), pluginsDir, processing.plugin, 'index.js'))
+  if (plugin.prepare && typeof plugin.prepare === 'function') {
+    const res = await (plugin.prepare as PrepareFunction)({ processingConfig: processing.config })
+    if (res.processingConfig) processing.config = res.processingConfig
+    if (res.secrets) {
+      processing.secrets = {}
+      Object.keys(res.secrets).forEach(key => {
+        processing.secrets![key] = cipher(res.secrets![key])
+      })
+    }
+  }
 }
 
 /**
@@ -55,6 +71,7 @@ const validateFullProcessing = async (processing: Processing) => {
  */
 const cleanProcessing = (processing: Processing, sessionState: SessionStateAuthenticated) => {
   delete processing.webhookKey
+  delete processing.secrets
   processing.userProfile = permissions.getUserResourceProfile(processing.owner, processing.permissions, sessionState)
   if (processing.userProfile !== 'admin') {
     for (const part of sensitiveParts) delete (processing as any)[part]
@@ -284,7 +301,7 @@ router.patch('/:id', async (req, res) => {
     date: new Date().toISOString()
   }
 
-  const patch: { $unset?: { [key: string]: true }, $set?: { [key: string]: any } } = {}
+  const patch: Record<string, any> = { $set: {} }
   for (const key in req.body) {
     if (req.body[key] === null) {
       patch.$unset = patch.$unset || {}
@@ -297,6 +314,11 @@ router.patch('/:id', async (req, res) => {
   }
   const patchedProcessing = { ...processing, ...req.body }
   await validateFullProcessing(patchedProcessing)
+  if (patchedProcessing.secrets) {
+    patch.$set.config = patchedProcessing.config
+    patch.$set.secrets = patchedProcessing.secrets
+  }
+
   await mongo.processings.updateOne({ _id: req.params.id }, patch)
   await mongo.runs.updateMany({ 'processing._id': processing._id }, { $set: { permissions: patchedProcessing.permissions || [] } })
   await applyProcessing(mongo, patchedProcessing)

api/src/utils/cipher.ts

@@ -0,0 +1,22 @@
+import config from '#config'
+import { createHash, randomBytes, createCipheriv } from 'node:crypto'
+
+export type CipheredContent = { iv: string, alg: 'aes256', data: string }
+
+// the secret key for cipher/decipher is a simple hash of config.cipherPassword
+const hash = createHash('sha256')
+hash.update(config.cipherPassword)
+const securityKey = hash.digest()
+
+export const cipher = (content: string): CipheredContent => {
+  const initVector = randomBytes(16)
+  const algo = 'aes256'
+  const cipher = createCipheriv(algo, securityKey, initVector)
+  let encryptedData = cipher.update(content, 'utf-8', 'hex')
+  encryptedData += cipher.final('hex')
+  return {
+    iv: initVector.toString('hex'),
+    alg: algo,
+    data: encryptedData
+  }
+}

api/types/processing/schema.js

@@ -90,6 +90,13 @@ export default {
         $ref: 'https://github.com/data-fair/processings/scheduling'
       }
     },
+    secrets: {
+      type: 'object',
+      readOnly: true,
+      additionalProperties: {
+        $ref: '#/$defs/cipheredContent'
+      }
+    },
     permissions: {
       type: 'array',
       title: 'Permissions',
@@ -147,5 +154,28 @@ export default {
       title: 'Clé pour exécution à distance du traitement',
       readOnly: true
     }
+  },
+  $defs: {
+    cipheredContent: {
+      type: 'object',
+      additionalProperties: false,
+      required: [
+        'iv',
+        'alg',
+        'data'
+      ],
+      properties: {
+        iv: {
+          type: 'string',
+        },
+        alg: {
+          type: 'string',
+          const: 'aes256'
+        },
+        data: {
+          type: 'string',
+        }
+      }
+    }
   }
 }

package.json

@@ -41,6 +41,7 @@
     "@data-fair/lib-types-builder": "^1.7.0"
   },
   "devDependencies": {
+    "@data-fair/lib-common-types": "^1.10.0",
     "@commitlint/cli": "^19.7.1",
     "@commitlint/config-conventional": "^19.7.1",
     "@data-fair/lib-node": "^2.4.0",

test-it/03-processings.ts

@@ -12,7 +12,7 @@ let plugin
 const createTestPlugin = async () => {
   plugin = (await superadmin.post('/api/v1/plugins', {
     name: '@data-fair/processing-hello-world',
-    version: '0.12.2',
+    version: '1.1.0',
     distTag: 'latest',
     description: 'Minimal plugin for data-fair-processings. Create one-line datasets on demand.'
   })).data
@@ -84,8 +84,8 @@ describe('processing', () => {
 
     const run = (await superadmin.get('/api/v1/runs/' + runs.results[0]._id)).data
     assert.equal(run.status, 'error')
-    assert.equal(run.log[0].type, 'step')
-    assert.equal(run.log[1].type, 'error')
+    assert.equal(run.log[2].type, 'step')
+    assert.equal(run.log[3].type, 'error')
 
     processing = (await superadmin.get(`/api/v1/processings/${processing._id}`)).data
     assert.ok(processing.lastRun)
@@ -120,7 +120,7 @@ describe('processing', () => {
     await testSpies.waitFor('isKilled', 10000)
     run = (await superadmin.get(`/api/v1/runs/${run._id}`)).data
     assert.equal(run.status, 'killed')
-    assert.equal(run.log.length, 4)
+    assert.equal(run.log.length, 6)
 
     // limits were updated
     const limits = (await superadmin.get('/api/v1/limits/user/superadmin')).data
@@ -157,7 +157,7 @@ describe('processing', () => {
     await testSpies.waitFor('isKilled', 10000)
     run = (await superadmin.get(`/api/v1/runs/${run._id}`)).data
     assert.equal(run.status, 'killed')
-    assert.equal(run.log.length, 2)
+    assert.equal(run.log.length, 4)
   })
 
   it('should fail a run if processings_seconds limit is exceeded', async () => {
@@ -221,4 +221,60 @@ describe('processing', () => {
     // failure is normal we have no api key
     assert.equal(runs.results[0].status, 'error')
   })
+
+  it('should config a new processing, with a secret field', async () => {
+    const processing = (await superadmin.post('/api/v1/processings', {
+      title: 'Hello processing',
+      plugin: plugin.id
+    })).data
+    assert.ok(processing._id)
+
+    // configure the processing
+    const patchRes = await superadmin.patch(`/api/v1/processings/${processing._id}`, {
+      active: true,
+      config: {
+        datasetMode: 'create',
+        dataset: { id: 'hello-world-test-processings', title: 'Hello world test processing' },
+        overwrite: false,
+        message: 'Hello world test processing',
+        secretField: 'my secret value'
+      }
+    })
+    assert.equal(patchRes.data.config.secretField, '********')
+
+    const getRes = await superadmin.get(`/api/v1/processings/${processing._id}`)
+    assert.equal(getRes.data.config.secretField, '********')
+
+    // Patch the processing to edit the secret field
+    const patchRes2 = await superadmin.patch(`/api/v1/processings/${processing._id}`, {
+      config: {
+        datasetMode: 'create',
+        dataset: { id: 'hello-world-test-processings', title: 'Hello world test processing' },
+        overwrite: false,
+        message: 'Hello world test processing',
+        secretField: 'my new secret value'
+      }
+    })
+    assert.equal(patchRes2.data.config.secretField, '********')
+
+    // trigger the processing
+    await Promise.all([
+      superadmin.post(`/api/v1/processings/${processing._id}/_trigger`),
+      testSpies.waitFor('isRunning', 10000)
+    ])
+
+    // nothing, failure is normal we have no api key
+    const [event] = await Promise.all([
+      testSpies.waitFor('pushEvent', 10000) as Promise<{ topic: { key: string } }>,
+      testSpies.waitFor('isFailure', 11000)
+    ])
+    assert.equal(event.topic.key, `processings:processing-finish-error:${processing._id}`)
+
+    const runs = (await superadmin.get('/api/v1/runs', { params: { processing: processing._id } })).data
+    assert.equal(runs.count, 1)
+    const run = (await superadmin.get('/api/v1/runs/' + runs.results[0]._id)).data
+    assert.equal(run.status, 'error')
+    assert.equal(run.log[1].type, 'info')
+    assert.equal(run.log[1].extra.secrets.secretField, 'my new secret value')
+  })
 })

worker/config/custom-environment-variables.cjs

@@ -1,4 +1,5 @@
 module.exports = {
+  cipherPassword: 'CIPHER_PASSWORD',
   dataDir: 'DATA_DIR',
   tmpDir: 'TMP_DIR',
   dataFairAPIKey: 'DATA_FAIR_API_KEY',

worker/config/default.cjs

@@ -1,4 +1,5 @@
 module.exports = {
+  cipherPassword: undefined,
   dataDir: '/app/data',
   tmpDir: null, // will be dataDir + '/tmp' if null
   dataFairAdminMode: false,

worker/config/development.cjs

@@ -1,4 +1,5 @@
 module.exports = {
+  cipherPassword: 'dev',
   dataDir: '../data/development',
   dataFairAdminMode: true,
   dataFairAPIKey: '', // override in local-development.cjs

worker/config/test.cjs

@@ -1,4 +1,5 @@
 module.exports = {
+  cipherPassword: 'test',
   dataDir: './data/test',
   dataFairAdminMode: true,
   dataFairAPIKey: 'dTpzdXBlcmFkbWluOjZEQ0NXY2ZrSHhVRVQxSzVudmNNg',