Add support for EXTENSIONS_LOCATION setting (directus#20207)

Co-authored-by: ian <[email protected]>
Co-authored-by: Brainslug <[email protected]>
Co-authored-by: Pascal Jufer <[email protected]>

Author: 4 people

.changeset/few-mice-know.md

@@ -0,0 +1,5 @@
+---
+'@directus/api': minor
+---
+
+Added configuration to allow extensions to be managed in a configured storage location

.eslintignore

@@ -9,7 +9,9 @@ dist/
 *.yml
 *.yaml
 *.md
+*.txt
 *.json
 *.scss
 *.css
 *.svg
+Dockerfile

Dockerfile

@@ -54,8 +54,6 @@ EXPOSE 8055
 ENV \
 	DB_CLIENT="sqlite3" \
 	DB_FILENAME="/directus/database/database.sqlite" \
-	EXTENSIONS_PATH="/directus/extensions" \
-	STORAGE_LOCAL_ROOT="/directus/uploads" \
 	NODE_ENV="production" \
 	NPM_CONFIG_UPDATE_NOTIFIER="false"
 

api/package.json

@@ -145,6 +145,7 @@
 		"openid-client": "5.4.2",
 		"ora": "6.3.1",
 		"otplib": "12.0.1",
+		"p-queue": "7.4.1",
 		"papaparse": "5.4.1",
 		"pino": "8.14.1",
 		"pino-http": "8.3.3",

api/src/database/index.ts

@@ -10,6 +10,7 @@ import path from 'path';
 import { performance } from 'perf_hooks';
 import { promisify } from 'util';
 import env from '../env.js';
+import { getExtensionsPath } from '../extensions/lib/get-extensions-path.js';
 import logger from '../logger.js';
 import type { DatabaseClient } from '../types/index.js';
 import { getConfigFromEnv } from '../utils/get-config-from-env.js';
@@ -258,7 +259,7 @@ export async function validateMigrations(): Promise<boolean> {
 	try {
 		let migrationFiles = await fse.readdir(path.join(__dirname, 'migrations'));
 
-		const customMigrationsPath = path.resolve(env['EXTENSIONS_PATH'], 'migrations');
+		const customMigrationsPath = path.resolve(getExtensionsPath(), 'migrations');
 
 		let customMigrationFiles =
 			((await fse.pathExists(customMigrationsPath)) && (await fse.readdir(customMigrationsPath))) || [];

api/src/database/migrations/run.ts

@@ -6,7 +6,7 @@ import { dirname } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import path from 'path';
 import { flushCaches } from '../../cache.js';
-import env from '../../env.js';
+import { getExtensionsPath } from '../../extensions/lib/get-extensions-path.js';
 import logger from '../../logger.js';
 import type { Migration } from '../../types/index.js';
 import getModuleDefault from '../../utils/get-module-default.js';
@@ -16,7 +16,7 @@ const __dirname = dirname(fileURLToPath(import.meta.url));
 export default async function run(database: Knex, direction: 'up' | 'down' | 'latest', log = true): Promise<void> {
 	let migrationFiles = await fse.readdir(__dirname);
 
-	const customMigrationsPath = path.resolve(env['EXTENSIONS_PATH'], 'migrations');
+	const customMigrationsPath = path.resolve(getExtensionsPath(), 'migrations');
 
 	let customMigrationFiles =
 		((await fse.pathExists(customMigrationsPath)) && (await fse.readdir(customMigrationsPath))) || [];

api/src/env.ts

@@ -3,8 +3,8 @@
  * For all possible keys, see: https://docs.directus.io/self-hosted/config-options/
  */
 
-import { parseJSON, toArray } from '@directus/utils';
 import { JAVASCRIPT_FILE_EXTS } from '@directus/constants';
+import { parseJSON, toArray } from '@directus/utils';
 import dotenv from 'dotenv';
 import fs from 'fs';
 import { clone, toNumber, toString } from 'lodash-es';
@@ -35,6 +35,7 @@ const allowedEnvironmentVars = [
 	'QUERY_LIMIT_MAX',
 	'QUERY_LIMIT_DEFAULT',
 	'ROBOTS_TXT',
+	'TEMP_PATH',
 	// server
 	'SERVER_.+',
 	// database
@@ -163,6 +164,7 @@ const allowedEnvironmentVars = [
 	'AUTH_.+_SP.+',
 	// extensions
 	'PACKAGE_FILE_LOCATION',
+	'EXTENSIONS_LOCATION',
 	'EXTENSIONS_PATH',
 	'EXTENSIONS_AUTO_RELOAD',
 	'EXTENSIONS_CACHE_TTL',
@@ -225,6 +227,8 @@ export const defaults: Record<string, any> = {
 	MAX_BATCH_MUTATION: Infinity,
 	ROBOTS_TXT: 'User-agent: *\nDisallow: /',
 
+	TEMP_PATH: './node_modules/.directus',
+
 	DB_EXCLUDE_TABLES: 'spatial_ref_sys,sysdiagrams',
 
 	STORAGE_LOCATIONS: 'local',

api/src/extensions/lib/get-extensions-path.ts

@@ -0,0 +1,10 @@
+import { join } from 'path';
+import env from '../../env.js';
+
+export const getExtensionsPath = () => {
+	if (env['EXTENSIONS_LOCATION']) {
+		return join(env['TEMP_PATH'], 'extensions');
+	}
+
+	return env['EXTENSIONS_PATH'];
+};

api/src/extensions/lib/get-extensions.ts

@@ -1,15 +1,16 @@
 import type { Extension } from '@directus/extensions';
 import { getLocalExtensions, getPackageExtensions, resolvePackageExtensions } from '@directus/extensions/node';
 import env from '../../env.js';
+import { getExtensionsPath } from './get-extensions-path.js';
 
 export const getExtensions = async () => {
-	const localExtensions = await getLocalExtensions(env['EXTENSIONS_PATH']);
+	const localExtensions = await getLocalExtensions(getExtensionsPath());
 
 	const loadedNames = localExtensions.map(({ name }) => name);
 
 	const filterDuplicates = ({ name }: Extension) => loadedNames.includes(name) === false;
 
-	const localPackageExtensions = (await resolvePackageExtensions(env['EXTENSIONS_PATH'])).filter((extension) =>
+	const localPackageExtensions = (await resolvePackageExtensions(getExtensionsPath())).filter((extension) =>
 		filterDuplicates(extension)
 	);
 

api/src/extensions/lib/sync-extensions.ts

@@ -0,0 +1,82 @@
+import { NESTED_EXTENSION_TYPES } from '@directus/extensions';
+import { ensureExtensionDirs } from '@directus/extensions/node';
+import mid from 'node-machine-id';
+import { createWriteStream } from 'node:fs';
+import { mkdir } from 'node:fs/promises';
+import { dirname, join, relative, resolve, sep } from 'node:path';
+import { pipeline } from 'node:stream/promises';
+import Queue from 'p-queue';
+import env from '../../env.js';
+import logger from '../../logger.js';
+import { getMessenger } from '../../messenger.js';
+import { getStorage } from '../../storage/index.js';
+import { getExtensionsPath } from './get-extensions-path.js';
+import { SyncStatus, getSyncStatus, setSyncStatus } from './sync-status.js';
+
+export const syncExtensions = async (): Promise<void> => {
+	const extensionsPath = getExtensionsPath();
+
+	if (!env['EXTENSIONS_LOCATION']) {
+		// Safe to run with multiple instances since dirs are created with `recursive: true`
+		return ensureExtensionDirs(extensionsPath, NESTED_EXTENSION_TYPES);
+	}
+
+	const messenger = getMessenger();
+
+	const isPrimaryProcess =
+		String(process.env['NODE_APP_INSTANCE']) === '0' || process.env['NODE_APP_INSTANCE'] === undefined;
+
+	const id = await mid.machineId();
+
+	const message = `extensions-sync/${id}`;
+
+	if (isPrimaryProcess === false) {
+		const isDone = (await getSyncStatus()) === SyncStatus.DONE;
+
+		if (isDone) return;
+
+		logger.trace('Extensions already being synced to this machine from another process.');
+
+		/**
+		 * Wait until the process that called the lock publishes a message that the syncing is complete
+		 */
+		return new Promise((resolve) => {
+			messenger.subscribe(message, () => resolve());
+		});
+	}
+
+	// Ensure that the local extensions cache path exists
+	await mkdir(extensionsPath, { recursive: true });
+	await setSyncStatus(SyncStatus.SYNCING);
+
+	logger.trace('Syncing extensions from configured storage location...');
+
+	const storage = await getStorage();
+
+	const disk = storage.location(env['EXTENSIONS_LOCATION']);
+
+	// Make sure we don't overload the file handles
+	const queue = new Queue({ concurrency: 1000 });
+
+	for await (const filepath of disk.list(env['EXTENSIONS_PATH'])) {
+		const readStream = await disk.read(filepath);
+
+		// We want files to be stored in the root of `$TEMP_PATH/extensions`, so gotta remove the
+		// extensions path on disk from the start of the file path
+		const destPath = join(extensionsPath, relative(resolve(sep, env['EXTENSIONS_PATH']), resolve(sep, filepath)));
+
+		// Ensure that the directory path exists
+		await mkdir(dirname(destPath), { recursive: true });
+
+		const writeStream = createWriteStream(destPath);
+
+		queue.add(() => pipeline(readStream, writeStream));
+	}
+
+	await queue.onIdle();
+
+	await ensureExtensionDirs(extensionsPath, NESTED_EXTENSION_TYPES);
+
+	await setSyncStatus(SyncStatus.DONE);
+	messenger.publish(message, { ready: true });
+};

api/src/extensions/lib/sync-status.ts

@@ -0,0 +1,29 @@
+import { exists } from 'fs-extra';
+import { readFile, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { getExtensionsPath } from './get-extensions-path.js';
+
+export enum SyncStatus {
+	UNKNOWN = 'UNKNOWN',
+	SYNCING = 'SYNCING',
+	DONE = 'DONE',
+}
+
+/**
+ * Retrieves the sync status from the `.status` file in the local extensions folder
+ */
+export const getSyncStatus = async () => {
+	const statusFilePath = join(getExtensionsPath(), '.status');
+
+	if (await exists(statusFilePath)) {
+		const status = await readFile(statusFilePath, 'utf8');
+		return status;
+	} else {
+		return SyncStatus.UNKNOWN;
+	}
+};
+
+export const setSyncStatus = async (status: SyncStatus) => {
+	const statusFilePath = join(getExtensionsPath(), '.status');
+	await writeFile(statusFilePath, status);
+};

api/src/extensions/manager.ts

@@ -10,7 +10,7 @@ import type {
 	OperationApiConfig,
 } from '@directus/extensions';
 import { APP_SHARED_DEPS, HYBRID_EXTENSION_TYPES, NESTED_EXTENSION_TYPES } from '@directus/extensions';
-import { ensureExtensionDirs, generateExtensionsEntrypoint } from '@directus/extensions/node';
+import { generateExtensionsEntrypoint } from '@directus/extensions/node';
 import type {
 	ActionHandler,
 	EmbedHandler,
@@ -45,11 +45,13 @@ import { getSchema } from '../utils/get-schema.js';
 import { importFileUrl } from '../utils/import-file-url.js';
 import { JobQueue } from '../utils/job-queue.js';
 import { scheduleSynchronizedJob, validateCron } from '../utils/schedule.js';
+import { getExtensionsPath } from './lib/get-extensions-path.js';
 import { getExtensionsSettings } from './lib/get-extensions-settings.js';
 import { getExtensions } from './lib/get-extensions.js';
 import { getSharedDepsMapping } from './lib/get-shared-deps-mapping.js';
 import { generateApiExtensionsSandboxEntrypoint } from './lib/sandbox/generate-api-extensions-sandbox-entrypoint.js';
 import { instantiateSandboxSdk } from './lib/sandbox/sdk/instantiate.js';
+import { syncExtensions } from './lib/sync-extensions.js';
 import { wrapEmbeds } from './lib/wrap-embeds.js';
 import type { BundleConfig, ExtensionManagerOptions } from './types.js';
 
@@ -173,7 +175,7 @@ export class ExtensionManager {
 	 */
 	private async load(): Promise<void> {
 		try {
-			await ensureExtensionDirs(env['EXTENSIONS_PATH'], NESTED_EXTENSION_TYPES);
+			await syncExtensions();
 
 			this.extensions = await getExtensions();
 			this.extensionsSettings = await getExtensionsSettings(this.extensions);
@@ -290,7 +292,7 @@ export class ExtensionManager {
 	private initializeWatcher(): void {
 		logger.info('Watching extensions for changes...');
 
-		const extensionDirUrl = pathToRelativeUrl(env['EXTENSIONS_PATH']);
+		const extensionDirUrl = pathToRelativeUrl(getExtensionsPath());
 
 		const localExtensionUrls = NESTED_EXTENSION_TYPES.flatMap((type) => {
 			const typeDir = path.posix.join(extensionDirUrl, pluralize(type));

api/src/services/mail/index.ts

@@ -1,3 +1,4 @@
+import { InvalidPayloadError } from '@directus/errors';
 import type { Accountability, SchemaOverview } from '@directus/types';
 import fse from 'fs-extra';
 import type { Knex } from 'knex';
@@ -7,7 +8,7 @@ import path from 'path';
 import { fileURLToPath } from 'url';
 import getDatabase from '../../database/index.js';
 import env from '../../env.js';
-import { InvalidPayloadError } from '@directus/errors';
+import { getExtensionsPath } from '../../extensions/lib/get-extensions-path.js';
 import logger from '../../logger.js';
 import getMailer from '../../mailer.js';
 import type { AbstractServiceOptions } from '../../types/index.js';
@@ -16,7 +17,7 @@ import { Url } from '../../utils/url.js';
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 const liquidEngine = new Liquid({
-	root: [path.resolve(env['EXTENSIONS_PATH'], 'templates'), path.resolve(__dirname, 'templates')],
+	root: [path.resolve(getExtensionsPath(), 'templates'), path.resolve(__dirname, 'templates')],
 	extname: '.liquid',
 });
 
@@ -81,7 +82,7 @@ export class MailService {
 	}
 
 	private async renderTemplate(template: string, variables: Record<string, any>) {
-		const customTemplatePath = path.resolve(env['EXTENSIONS_PATH'], 'templates', template + '.liquid');
+		const customTemplatePath = path.resolve(getExtensionsPath(), 'templates', template + '.liquid');
 		const systemTemplatePath = path.join(__dirname, 'templates', template + '.liquid');
 
 		const templatePath = (await fse.pathExists(customTemplatePath)) ? customTemplatePath : systemTemplatePath;

api/src/utils/validate-storage.ts

@@ -1,9 +1,10 @@
-import env from '../env.js';
-import logger from '../logger.js';
-import { access } from 'node:fs/promises';
+import { toArray } from '@directus/utils';
 import { constants } from 'fs';
+import { access } from 'node:fs/promises';
 import path from 'path';
-import { toArray } from '@directus/utils';
+import env from '../env.js';
+import { getExtensionsPath } from '../extensions/lib/get-extensions-path.js';
+import logger from '../logger.js';
 
 export async function validateStorage(): Promise<void> {
 	if (env['DB_CLIENT'] === 'sqlite3') {
@@ -28,9 +29,11 @@ export async function validateStorage(): Promise<void> {
 		}
 	}
 
-	try {
-		await access(env['EXTENSIONS_PATH'], constants.R_OK);
-	} catch {
-		logger.warn(`Extensions directory (${path.resolve(env['EXTENSIONS_PATH'])}) is not readable!`);
+	if (!env['EXTENSIONS_LOCATION']) {
+		try {
+			await access(getExtensionsPath(), constants.R_OK);
+		} catch {
+			logger.warn(`Extensions directory (${path.resolve(getExtensionsPath())}) is not readable!`);
+		}
 	}
 }

docs/self-hosted/config-options.md

@@ -244,6 +244,7 @@ prefixing the value with `{type}:`. The following types are available:
 | `QUERY_LIMIT_DEFAULT`      | The default query limit used when not defined in the API request.                                                           | `100`                        |
 | `QUERY_LIMIT_MAX`          | The maximum query limit accepted on API requests.                                                                           | `-1`                         |
 | `ROBOTS_TXT`               | What the `/robots.txt` endpoint should return                                                                               | `User-agent: *\nDisallow: /` |
+| `TEMP_PATH`                | Where Directus' temporary files should be managed                                                                           | `./node_modules/.directus`   |
 
 <sup>[1]</sup> The PUBLIC_URL value is used for things like OAuth redirects, forgot-password emails, and logos that
 needs to be publicly available on the internet.
@@ -898,19 +899,29 @@ const publicUrl = process.env.PUBLIC_URL;
 
 ## Extensions
 
-| Variable                             | Description                                             | Default Value  |
-| ------------------------------------ | ------------------------------------------------------- | -------------- |
-| `EXTENSIONS_PATH`                    | Path to your local extensions folder.                   | `./extensions` |
-| `EXTENSIONS_AUTO_RELOAD`             | Automatically reload extensions when they have changed. | `false`        |
-| `EXTENSIONS_CACHE_TTL`<sup>[1]</sup> | How long custom app Extensions get cached by browsers.  | --             |
+| Variable                               | Description                                             | Default Value  |
+| -------------------------------------- | ------------------------------------------------------- | -------------- |
+| `EXTENSIONS_PATH`<sup>[1]</sup>        | Path to your local extensions folder.                   | `./extensions` |
+| `EXTENSIONS_AUTO_RELOAD`<sup>[2]</sup> | Automatically reload extensions when they have changed. | `false`        |
+| `EXTENSIONS_CACHE_TTL`<sup>[3]</sup>   | How long custom app Extensions get cached by browsers.  | --             |
+| `EXTENSIONS_LOCATION`<sup>[4]</sup>    | What configured storage location to use for extensions. | --             |
 
-<sup>[1]</sup> The `EXTENSIONS_CACHE_TTL` environment variable controls for how long custom app extensions (e.t.,
+<sup>[1]</sup> If `EXTENSIONS_LOCATION` is configured, this is the path to the extensions folder within the selected
+storage location.
+
+<sup>[2]</sup> `EXTENSIONS_AUTO_RELOAD` will not work when the `EXTENSION_LOCATION` environment variable is set.
+
+<sup>[3]</sup> The `EXTENSIONS_CACHE_TTL` environment variable controls for how long custom app extensions (e.t.,
 interface, display, layout, module, panel) are cached by browsers. Caching can speed-up the loading of the app as the
 code for the extensions doesn't need to be re-fetched from the server on each app reload. On the other hand, this means
 that code changes to app extensions won't be taken into account by the browser until `EXTENSIONS_CACHE_TTL` has expired.
 By default, extensions are not cached. The input data type for this environment variable is the same as
 [`CACHE_TTL`](#cache).
 
+<sup>[4]</sup> By default extensions are loaded from the local file system. `EXTENSIONS_LOCATION` can be used to load
+extensions from a storage location instead. Under the hood, they are synced into a local directory within `TEMP_PATH`
+and then loaded from there.
+
 ## Messenger
 
 | Variable              | Description                            | Default Value        |
@@ -1052,14 +1063,16 @@ These environment variables only exist when you're using the official Docker Con
 For more information on what these options do, please refer to
 [the `pm2` documentation](https://pm2.keymetrics.io/docs/usage/application-declaration/).
 
-| Variable                 | Description                                                        | Default     |
-| ------------------------ | ------------------------------------------------------------------ | ----------- |
-| `PM2_INSTANCES`          | Number of app instance to be launched                              | `1`         |
-| `PM2_EXEC_MODE`          | One of `fork`, `cluster`                                           | `'cluster'` |
-| `PM2_MAX_MEMORY_RESTART` | App will be restarted if it exceeds the amount of memory specified | —           |
-| `PM2_MIN_UPTIME`         | Min uptime of the app to be considered started                     | —           |
-| `PM2_LISTEN_TIMEOUT`     | Time in ms before forcing a reload if app not listening            | —           |
-| `PM2_KILL_TIMEOUT`       | Time in milliseconds before sending a final SIGKILL                | —           |
-| `PM2_MAX_RESTARTS`       | Number of failed restarts before the process is killed             |  —          |
-| `PM2_RESTART_DELAY`      | Time to wait before restarting a crashed app                       | `0`         |
-| `PM2_AUTO_RESTART`       | Automatically restart Directus if it crashes unexpectedly          | `false`     |
+| Variable                      | Description                                                        | Default     |
+| ----------------------------- | ------------------------------------------------------------------ | ----------- |
+| `PM2_INSTANCES`<sup>[1]</sup> | Number of app instance to be launched                              | `1`         |
+| `PM2_EXEC_MODE`               | One of `fork`, `cluster`                                           | `'cluster'` |
+| `PM2_MAX_MEMORY_RESTART`      | App will be restarted if it exceeds the amount of memory specified | —           |
+| `PM2_MIN_UPTIME`              | Min uptime of the app to be considered started                     | —           |
+| `PM2_LISTEN_TIMEOUT`          | Time in ms before forcing a reload if app not listening            | —           |
+| `PM2_KILL_TIMEOUT`            | Time in milliseconds before sending a final SIGKILL                | —           |
+| `PM2_MAX_RESTARTS`            | Number of failed restarts before the process is killed             |  —          |
+| `PM2_RESTART_DELAY`           | Time to wait before restarting a crashed app                       | `0`         |
+| `PM2_AUTO_RESTART`            | Automatically restart Directus if it crashes unexpectedly          | `false`     |
+
+<sup>[1]</sup> [Redis](#redis) is required in case of multiple instances.

packages/storage-driver-s3/src/index.ts

@@ -200,12 +200,17 @@ export class DriverS3 implements Driver {
 	}
 
 	async *list(prefix = '') {
+		let Prefix = this.fullPath(prefix);
+
+		// Current dir (`.`) isn't known to S3, needs to be an empty prefix instead
+		if (Prefix === '.') Prefix = '';
+
 		let continuationToken: string | undefined = undefined;
 
 		do {
 			const listObjectsV2CommandInput: ListObjectsV2CommandInput = {
 				Bucket: this.config.bucket,
-				Prefix: this.fullPath(prefix),
+				Prefix,
 				MaxKeys: 1000,
 			};
 
@@ -218,10 +223,14 @@ export class DriverS3 implements Driver {
 			continuationToken = response.NextContinuationToken;
 
 			if (response.Contents) {
-				for (const file of response.Contents) {
-					if (file.Key) {
-						yield file.Key.substring(this.root.length);
-					}
+				for (const object of response.Contents) {
+					if (!object.Key) continue;
+
+					const isDir = object.Key.endsWith('/');
+
+					if (isDir) continue;
+
+					yield object.Key.substring(this.root.length);
 				}
 			}
 		} while (continuationToken);