Support export to JSON, CSV

Signed-off-by: Anna Khismatullina <[email protected]>
hcengineering · Mar 7, 2025 · e9a3fad · e9a3fad
1 parent ea5576f
commit e9a3fad
Show file tree

Hide file tree

Showing 46 changed files with 2,021 additions and 78 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -759,5 +759,24 @@
       "outputCapture": "std",
       "cwd": "${workspaceRoot}/dev/import-tool"
     },
+    {
+      "name": "Debug Export service",
+      "type": "node",
+      "request": "launch",
+      "args": ["src/index.ts"],
+      "env": {
+        "PORT": "4009",
+        "SECRET": "secret",
+        "ACCOUNTS_URL": "http://localhost:3000",
+        "MINIO_ENDPOINT": "localhost",
+        "MINIO_ACCESS_KEY": "minioadmin",
+        "MINIO_SECRET_KEY": "minioadmin",
+        "TRANSACTOR_URL": "ws://localhost:3333",
+        "SERVICE_ID": "export-service"
+      },
+      "runtimeArgs": ["--nolazy", "-r", "ts-node/register"],
+      "sourceMaps": true,
+      "cwd": "${workspaceRoot}/services/export/pod-export"
+    }
   ]
 }
diff --git a/common/config/rush/command-line.json b/common/config/rush/command-line.json
@@ -238,7 +238,7 @@
       "summary": "Build docker with platform",
       "description": "use to build all docker containers required for platform",
       "safeForSimultaneousRushProcesses": true,
-      "shellCommand": "rush docker:build -p 20 --to @hcengineering/pod-server --to @hcengineering/pod-front --to @hcengineering/prod --to @hcengineering/pod-account --to @hcengineering/pod-workspace --to @hcengineering/pod-collaborator --to @hcengineering/tool --to @hcengineering/pod-print --to @hcengineering/pod-sign --to @hcengineering/pod-analytics-collector --to @hcengineering/rekoni-service --to @hcengineering/pod-ai-bot --to @hcengineering/import-tool --to @hcengineering/pod-stats --to @hcengineering/pod-fulltext --to @hcengineering/pod-love --to @hcengineering/green --to @hcengineering/pod-mail"
+      "shellCommand": "rush docker:build -p 20 --to @hcengineering/pod-server --to @hcengineering/pod-front --to @hcengineering/prod --to @hcengineering/pod-account --to @hcengineering/pod-workspace --to @hcengineering/pod-collaborator --to @hcengineering/tool --to @hcengineering/pod-print --to @hcengineering/pod-sign --to @hcengineering/pod-analytics-collector --to @hcengineering/rekoni-service --to @hcengineering/pod-ai-bot --to @hcengineering/import-tool --to @hcengineering/pod-stats --to @hcengineering/pod-fulltext --to @hcengineering/pod-love --to @hcengineering/green --to @hcengineering/pod-mail --to @hcengineering/pod-export"
     },
     {
       "commandKind": "global",

diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml
diff --git a/desktop/src/ui/platform.ts b/desktop/src/ui/platform.ts
@@ -351,6 +351,7 @@ export async function configurePlatform (): Promise<void> {
   configureNotifications()
 
   setMetadata(setting.metadata.BackupUrl, config.BACKUP_URL ?? '')
+  setMetadata(setting.metadata.ExportUrl, config.EXPORT_URL ?? '')
 
   if (config.INITIAL_URL !== '') {
     setLocationStorageKey('uberflow_child')

diff --git a/desktop/src/ui/types.ts b/desktop/src/ui/types.ts
@@ -38,8 +38,8 @@ export interface Config {
   UPLOAD_CONFIG: string
   UPLOAD_URL: string
   VERSION?: string
-
   BACKUP_URL?: string
+  EXPORT_URL?: string
 }
 
 export interface Branding {

diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml
@@ -519,6 +519,24 @@ services:
 #      - ACCOUNTS_URL=http://host.docker.internal:3000
 #      - SERVICE_ID=telegram-bot-service
 #      - STATS_URL=http://host.docker.internal:4900
+  export:
+    image: hardcoreeng/export
+    extra_hosts:
+      - 'host.docker.internal:host-gateway'
+    links:
+      - mongodb
+      - minio
+      - stats
+    ports:
+      - 4009:4009
+    environment:
+      - PORT=4009
+      - SECRET=secret
+      - SERVICE_ID=export-service
+      - DB_URL=${MONGO_URL}
+      - STATS_URL=http://host.docker.internal:4900
+      - STORAGE_CONFIG=${STORAGE_CONFIG}
+      - ACCOUNTS_URL=http://host.docker.internal:3000
 volumes:
   db:
   dbpg:

diff --git a/dev/import-tool/src/index.ts b/dev/import-tool/src/index.ts
@@ -17,11 +17,11 @@ import {
   ClickupImporter,
   defaultDocumentPreprocessors,
   DocumentConverter,
-  FrontFileUploader,
-  importNotion,
-  UnifiedFormatImporter,
   type DocumentConverterOptions,
   type FileUploader,
+  FrontFileUploader,
+  HulyFormatImporter,
+  importNotion,
   type Logger
 } from '@hcengineering/importer'
 import { setMetadata } from '@hcengineering/platform'
@@ -167,7 +167,7 @@ export function importTool (): void {
     .action(async (dir: string, cmd) => {
       const { workspace, user, password } = cmd
       await authorize(user, password, workspace, async (client, uploader) => {
-        const importer = new UnifiedFormatImporter(client, uploader, new ConsoleLogger())
+        const importer = new HulyFormatImporter(client, uploader, new ConsoleLogger())
         await importer.importFolder(dir)
       })
     })

diff --git a/dev/prod/public/config.json b/dev/prod/public/config.json
@@ -18,5 +18,6 @@
   "VERSION": null,
   "MODEL_VERSION": null,
   "STATS_URL": "http://localhost:4900",
-  "PASSWORD_STRICTNESS": "none"
+  "PASSWORD_STRICTNESS": "none",
+  "EXPORT_URL": "http://localhost:4009/export"
 }
diff --git a/dev/prod/src/platform.ts b/dev/prod/src/platform.ts
@@ -169,6 +169,7 @@ export interface Config {
   USE_BINARY_PROTOCOL?: boolean
   TRANSACTOR_OVERRIDE?: string
   BACKUP_URL?: string
+  EXPORT_URL?: string
 }
 
 export interface Branding {
@@ -534,6 +535,7 @@ export async function configurePlatform() {
   setMetadata(workbench.metadata.DefaultSpecial, myBranding.defaultSpecial ?? 'issues')
 
   setMetadata(setting.metadata.BackupUrl, config.BACKUP_URL ?? '')
+  setMetadata(setting.metadata.ExportUrl, config.EXPORT_URL ?? '')
 
   initThemeStore()
 }
diff --git a/models/setting/src/index.ts b/models/setting/src/index.ts
@@ -336,6 +336,20 @@ export function createModel (builder: Builder): void {
     },
     setting.ids.InviteSettings
   )
+  builder.createDoc(
+    setting.class.WorkspaceSettingCategory,
+    core.space.Model,
+    {
+      name: 'export',
+      label: setting.string.Export,
+      icon: setting.icon.Export,
+      component: setting.component.Export,
+      group: 'settings-editor',
+      role: AccountRole.User,
+      order: 4800
+    },
+    setting.ids.Export
+  )
   // Currently remove Support item from settings
   // builder.createDoc(
   //   setting.class.SettingsCategory,

diff --git a/packages/importer/src/docx/docx.ts b/packages/importer/src/docx/docx.ts
@@ -17,7 +17,7 @@ import { defaultExtensions, htmlToJSON, MarkupNode, serializeMessage } from '@hc
 import { mkdir, readdir, readFile, writeFile } from 'fs/promises'
 import * as yaml from 'js-yaml'
 import { basename, dirname, extname, join, relative } from 'path'
-import { UnifiedControlledDocumentHeader, UnifiedDocumentTemplateHeader } from '../huly/unified'
+import { HulyControlledDocumentHeader, HulyDocumentTemplateHeader } from '../huly/huly'
 
 export interface DocumentConverterOptions {
   outputPath: string
@@ -31,7 +31,7 @@ export interface DocumentState {
   path: string
   root: string
   markup: MarkupNode
-  header?: UnifiedControlledDocumentHeader | UnifiedDocumentTemplateHeader
+  header?: HulyControlledDocumentHeader | HulyDocumentTemplateHeader
 }
 
 export interface DocumentPreprocessorOptions<T> {