feat: カスタム絵文字エクスポート

CHANGELOG.md

@@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - クライアント: メンションにユーザーのアバターを表示するように
 - インスタンスプロフィールレンダリング ready
+- カスタム絵文字エクスポート機能
 
 ### Fixed
 

package.json

@@ -117,6 +117,7 @@
 		"@typescript-eslint/parser": "5.8.0",
 		"@vue/composition-api": "1.4.3",
 		"abort-controller": "3.0.0",
+		"archiver": "5.3.0",
 		"animejs": "3.2.1",
 		"apexcharts": "3.32.1",
 		"autobind-decorator": "2.4.0",
@@ -186,9 +187,10 @@
 		"langmap": "0.0.16",
 		"loader-utils": "2.0.0",
 		"lodash": "4.17.21",
+		"mime-types": "2.1.34",
 		"mocha": "9.1.3",
 		"moji": "0.5.1",
-		"ms": "2.1.3",
+		"ms": "3.0.0-canary.1",
 		"multer": "1.4.4",
 		"nested-property": "4.0.0",
 		"node-fetch": "2.6.6",

src/misc/download-url.ts

@@ -11,7 +11,7 @@ const PrivateIp = require('private-ip');
 
 const pipeline = util.promisify(stream.pipeline);
 
-export async function downloadUrl(url: string, path: string) {
+export async function downloadUrl(url: string, path: string): Promise<void> {
 	const logger = new Logger('download');
 
 	logger.info(`Downloading ${chalk.cyan(url)} ...`);

src/queue/index.ts

@@ -13,6 +13,7 @@ import { DriveFile } from '../models/entities/drive-file';
 import { getJobInfo } from './get-job-info';
 import { IActivity } from '../remote/activitypub/type';
 import { dbQueue, deliverQueue, inboxQueue, objectStorageQueue } from './queues';
+import { ThinUser } from './types';
 
 function renderError(e: Error): any {
 	return {
@@ -108,6 +109,15 @@ export function createDeleteDriveFilesJob(user: ILocalUser) {
 	});
 }
 
+export function createExportCustomEmojisJob(user: ILocalUser) {
+	return dbQueue.add('exportCustomEmojis', {
+		user: user,
+	}, {
+		removeOnComplete: true,
+		removeOnFail: true,
+	});
+}
+
 export function createExportNotesJob(user: ILocalUser) {
 	return dbQueue.add('exportNotes', {
 		user: user

src/queue/processors/db/export-custom-emojis.ts

@@ -0,0 +1,119 @@
+import * as Bull from 'bull';
+import * as tmp from 'tmp';
+import * as fs from 'fs';
+
+import { ulid } from 'ulid';
+const mime = require('mime-types');
+const archiver = require('archiver');
+import { queueLogger } from '../../logger';
+import addFile from '@/services/drive/add-file';
+import * as dateFormat from 'dateformat';
+import { Users, Emojis } from '@/models/index';
+import {  } from '@/queue/types';
+import { downloadUrl } from '@/misc/download-url';
+
+const logger = queueLogger.createSubLogger('export-custom-emojis');
+
+export async function exportCustomEmojis(job: Bull.Job, done: () => void): Promise<void> {
+	logger.info(`Exporting custom emojis ...`);
+
+	const user = await Users.findOne(job.data.user.id);
+	if (user == null) {
+		done();
+		return;
+	}
+
+	// Create temp dir
+	const [path, cleanup] = await new Promise<[string, () => void]>((res, rej) => {
+		tmp.dir((e, path, cleanup) => {
+			if (e) return rej(e);
+			res([path, cleanup]);
+		});
+	});
+
+	logger.info(`Temp dir is ${path}`);
+
+	const metaPath = path + '/meta.json';
+
+	fs.writeFileSync(metaPath, '', 'utf-8');
+
+	const metaStream = fs.createWriteStream(metaPath, { flags: 'a' });
+
+	const writeMeta = (text: string): Promise<void> => {
+		return new Promise<void>((res, rej) => {
+			metaStream.write(text, err => {
+				if (err) {
+					logger.error(err);
+					rej(err);
+				} else {
+					res();
+				}
+			});
+		});
+	};
+
+	await writeMeta(`{"metaVersion":1,"emojis":[`);
+
+	const customEmojis = await Emojis.find({
+		where: {
+			host: null,
+		},
+		order: {
+			id: 'ASC',
+		},
+	});
+
+	for (const emoji of customEmojis) {
+		const exportId = ulid().toLowerCase();
+		const ext = mime.extension(emoji.type);
+		const emojiPath = path + '/' + exportId + (ext ? '.' + ext : '');
+		fs.writeFileSync(emojiPath, '', 'binary');
+		let downloaded = false;
+
+		try {
+			await downloadUrl(emoji.url, emojiPath);
+			downloaded = true;
+		} catch (e) { // TODO: 何度か再試行
+			logger.error(e);
+		}
+
+		const content = JSON.stringify({
+			id: exportId,
+			downloaded: downloaded,
+			emoji: emoji,
+		});
+		const isFirst = customEmojis.indexOf(emoji) === 0;
+
+		await writeMeta(isFirst ? content : ',\n' + content);
+	}
+
+	await writeMeta(']}');
+
+	metaStream.end();
+
+	// Create archive
+	const [archivePath, archiveCleanup] = await new Promise<[string, () => void]>((res, rej) => {
+		tmp.file((e, path, fd, cleanup) => {
+			if (e) return rej(e);
+			res([path, cleanup]);
+		});
+	});
+	const archiveStream = fs.createWriteStream(archivePath);
+	const archive = archiver('zip', {
+		zlib: { level: 0 },
+	});
+	archiveStream.on('close', async () => {
+		logger.succ(`Exported to: ${archivePath}`);
+
+		const fileName = 'custom-emojis-' + dateFormat(new Date(), 'yyyy-mm-dd-HH-MM-ss') + '.zip';
+		const driveFile = await addFile(user, archivePath, fileName, null, null, true);
+
+		logger.succ(`Exported to: ${driveFile.id}`);
+		cleanup();
+		archiveCleanup();
+		done();
+	});
+	archive.pipe(archiveStream);
+	archive.directory(path, false);
+	archive.finalize();
+}

src/queue/processors/db/export-notes.ts

@@ -35,16 +35,20 @@ export async function exportNotes(job: Bull.Job<DbUserJobData>, done: any): Prom
 
 	const stream = fs.createWriteStream(path, { flags: 'a' });
 
-	await new Promise((res, rej) => {
-		stream.write('[', err => {
-			if (err) {
-				logger.error(err);
-				rej(err);
-			} else {
-				res();
-			}
+	const write = (text: string): Promise<void> => {
+		return new Promise<void>((res, rej) => {
+			stream.write(text, err => {
+				if (err) {
+					logger.error(err);
+					rej(err);
+				} else {
+					res();
+				}
+			});
 		});
-	});
+	};
+
+	await write('[');
 
 	let exportedNotesCount = 0;
 	let cursor: any = null;
@@ -74,16 +78,8 @@ export async function exportNotes(job: Bull.Job<DbUserJobData>, done: any): Prom
 				poll = await Polls.findOne({ noteId: note.id }).then(ensure);
 			}
 			const content = JSON.stringify(serialize(note, poll));
-			await new Promise((res, rej) => {
-				stream.write(exportedNotesCount === 0 ? content : ',\n' + content, err => {
-					if (err) {
-						logger.error(err);
-						rej(err);
-					} else {
-						res();
-					}
-				});
-			});
+			const isFirst = exportedNotesCount === 0;
+			await write(isFirst ? content : ',\n' + content);
 			exportedNotesCount++;
 		}
 
@@ -94,16 +90,7 @@ export async function exportNotes(job: Bull.Job<DbUserJobData>, done: any): Prom
 		job.progress(exportedNotesCount / total);
 	}
 
-	await new Promise((res, rej) => {
-		stream.write(']', err => {
-			if (err) {
-				logger.error(err);
-				rej(err);
-			} else {
-				res();
-			}
-		});
-	});
+	await write(']');
 
 	stream.end();
 	logger.succ(`Exported to: ${path}`);

src/queue/processors/db/index.ts

@@ -1,5 +1,7 @@
 import * as Bull from 'bull';
+import { DbJobData } from '../../types';
 import { deleteDriveFiles } from './delete-drive-files';
+import { exportCustomEmojis } from './export-custom-emojis';
 import { exportNotes } from './export-notes';
 import { exportFollowing } from './export-following';
 import { exportMute } from './export-mute';
@@ -9,10 +11,10 @@ import { importFollowing } from './import-following';
 import { importBlocking } from './import-blocking';
 import { importUserLists } from './import-user-lists';
 import { deleteAccount } from './delete-account';
-import { DbJobData } from '../../types';
 
 const jobs = {
 	deleteDriveFiles,
+	exportCustomEmojis,
 	exportNotes,
 	exportFollowing,
 	exportMute,

src/server/api/endpoints/export-custom-emojis.ts

@@ -0,0 +1,17 @@
+import $ from 'cafy';
+import define from '../define';
+import { createExportCustomEmojisJob } from '@/queue/index';
+import ms from 'ms';
+
+export const meta = {
+	secure: true,
+	requireCredential: true as const,
+	limit: {
+		duration: ms('1hour'),
+		max: 1,
+	},
+};
+
+export default define(meta, async (ps, user) => {
+	createExportCustomEmojisJob(user);
+});