feat(runtime): add feature support runing in docker env

runtimes/nodejs/Dockerfile

@@ -16,9 +16,11 @@ ENV FORCE_COLOR=1
 COPY . /app
 # COPY --chown=node:node . /app
 RUN mkdir /app/data || true
+RUN mkdir /app/cloud_functions || true
 RUN mkdir /tmp/custom_dependency || true
 RUN chown node:node /app/data
 RUN chown node:node /app/functions
+RUN chown -R node:node /app/cloud_functions
 RUN chown node:node /tmp/custom_dependency
 # RUN npm install
 # RUN npm run build
@@ -27,4 +29,4 @@ RUN chown node:node /app/package.json
 RUN chown node:node /app/package-lock.json
 
 USER node
-CMD [ "sh", "/app/start.sh" ]
+CMD [ "sh", "/app/start.sh" ]

runtimes/nodejs/package.json

@@ -52,6 +52,7 @@
     "nodemailer": "^6.6.3",
     "pako": "^2.1.0",
     "source-map-support": "^0.5.21",
+    "tar": "^6.2.1",
     "typescript-language-server": "^3.3.2",
     "validator": "^13.7.0",
     "vscode-languageserver": "^9.0.1",
@@ -87,4 +88,4 @@
   "lint-staged": {
     "*.{ts,js}": "eslint --fix"
   }
-}
+}

runtimes/nodejs/src/config.ts

@@ -120,4 +120,12 @@ export default class Config {
   static get CUSTOM_DEPENDENCY_BASE_PATH(): string {
     return process.env.CUSTOM_DEPENDENCY_BASE_PATH || '/tmp/custom_dependency'
   }
+
+  static get IS_DOCKER_PRODUCT(): boolean {
+    return process.env.DOCKER_PRODUCT === 'true'
+  }
+
+  static get DOCKER_PRODUCT_MONGO(): boolean {
+    return process.env.DOCKER_PRODUCT_MONGO === 'true'
+  }
 }

runtimes/nodejs/src/db.ts

@@ -31,6 +31,10 @@ export class DatabaseAgent {
   }
 
   static async initialize() {
+    if (Config.IS_DOCKER_PRODUCT && Config.DOCKER_PRODUCT_MONGO === false) {
+      return
+    }
+
     const client = new MongoClient(Config.DB_URI)
 
     let retryDelay = 1000 // 1s

runtimes/nodejs/src/handler/docker-file-handler.ts

@@ -0,0 +1,194 @@
+import { Response } from 'express'
+import { createReadStream } from 'fs'
+import fs from 'fs/promises'
+import fsPromises from 'fs/promises'
+import os from 'os'
+import path from 'path'
+import tar from 'tar'
+import { pipeline } from 'stream/promises'
+import { DatabaseAgent } from '../db'
+import { CLOUD_FUNCTION_COLLECTION, CONFIG_COLLECTION } from '../constants'
+import { ICloudFunctionData } from '../support/engine/types'
+import { IRequest } from '../support/types'
+
+// Constants for file paths
+const WORKSPACE_PATH = path.join(__dirname, '../../cloud_functions')
+
+const TEMP_DIR = path.join(os.tmpdir(), 'cloud_functions')
+const TAR_FILE_NAME = 'cloud_functions.tar'
+const TAR_FILE_PATH = path.join(TEMP_DIR, TAR_FILE_NAME)
+
+// Function to write cloud function data to the workspace as JSON files
+async function saveFunctionsToWorkspace(): Promise<void> {
+  // Check if the workspace directory exists and delete it if it does
+  try {
+    await fs.rm(WORKSPACE_PATH, { recursive: true, force: true })
+  } catch (err) {
+    console.error('Error removing the workspace directory:', err)
+  }
+
+  await fs.mkdir(WORKSPACE_PATH, { recursive: true })
+
+  const funcs = await DatabaseAgent.db
+    .collection<ICloudFunctionData>(CLOUD_FUNCTION_COLLECTION)
+    .find()
+    .toArray()
+
+  for (const func of funcs) {
+    const dirPath = path.join(WORKSPACE_PATH, path.dirname(func.name))
+    await fs.mkdir(dirPath, { recursive: true })
+    const filePath = path.join(WORKSPACE_PATH, func.name)
+    await fs.writeFile(filePath, JSON.stringify(func), 'utf8')
+  }
+}
+
+// Function to create a tarball of the workspace directory
+async function createTarPackage(): Promise<void> {
+  // Check if the tar file exists and delete it if it does
+  try {
+    await fs.rm(TAR_FILE_PATH, { force: true })
+  } catch (err) {
+    console.error('Error removing the tar file:', err)
+  }
+
+  await fs.mkdir(TEMP_DIR, { recursive: true })
+
+  await tar.create(
+    {
+      gzip: false,
+      file: TAR_FILE_PATH,
+      cwd: WORKSPACE_PATH,
+    },
+    ['.'],
+  )
+}
+
+// /docker-file
+export async function handleCloudFunctionTarPackage(
+  _: IRequest,
+  res: Response,
+) {
+  try {
+    // Save them to the workspace directory
+    await saveFunctionsToWorkspace()
+    // Create a tarball of the workspace
+    await createTarPackage()
+
+    // Set headers to serve the tarball as a download
+    res.writeHead(200, {
+      'Content-Type': 'application/x-tar',
+      'Content-Disposition': `attachment; filename="${TAR_FILE_NAME}"`,
+    })
+    // create file stream
+    const readStream = createReadStream(TAR_FILE_PATH)
+
+    // use pipeline pipe the file stream to response
+    await pipeline(readStream, res).then(() => {
+      // after pipe the file stream, remove the tar file
+      fsPromises.unlink(TAR_FILE_PATH).catch((unlinkError) => {
+        console.error('Error removing file:', unlinkError)
+      })
+    })
+  } catch (error) {
+    console.error('Error:', error)
+    if (!res.headersSent) {
+      return res.status(500).send('Internal Server Error')
+    }
+  }
+}
+
+export async function handleDockerFile(_: IRequest, res: Response) {
+  try {
+    const ENV = process.env
+    const keysToInclude = [
+      'DB_URI',
+      'APP_ID',
+      'APPID',
+      'RUNTIME_DOMAIN',
+      'RUNTIME_MAIN_IMAGE',
+
+      'OSS_ACCESS_KEY',
+      'OSS_ACCESS_SECRET',
+      'OSS_INTERNAL_ENDPOINT',
+      'OSS_EXTERNAL_ENDPOINT',
+      'OSS_REGION',
+
+      'DEPENDENCIES',
+
+      'NODE_MODULES_PUSH_URL',
+      'NODE_MODULES_PULL_URL',
+
+      'NPM_INSTALL_FLAGS',
+      'CUSTOM_DEPENDENCY_BASE_PATH',
+      'RESTART_AT',
+      'SERVER_SECRET',
+    ]
+
+    let envVariablesString = 'LOG_LEVEL=debug \\\n  FORCE_COLOR=1 \\\n  '
+
+    envVariablesString += 'DOCKER_PRODUCT=true \\\n  '
+    envVariablesString += 'DOCKER_PRODUCT_MONGO=false \\\n  '
+
+    const conf = await DatabaseAgent.db
+      .collection(CONFIG_COLLECTION)
+      .findOne({})
+
+    if (conf && conf.environments) {
+      for (const env of conf.environments) {
+        envVariablesString += `${env.name}=${env.value} \\\n  `
+      }
+    }
+
+    for (const [key, value] of Object.entries(ENV)) {
+      if (keysToInclude.includes(key)) {
+        envVariablesString += `${key}=${value} \\\n  `
+      }
+    }
+
+    envVariablesString = envVariablesString.replace(/\\\n\s*$/, '')
+
+    // version from env  todo
+    const DOCKER_FILE = `
+
+FROM ${ENV.RUNTIME_MAIN_IMAGE} as builder
+USER root
+
+WORKDIR ${ENV.CUSTOM_DEPENDENCY_BASE_PATH}
+RUN mkdir -p /app/cloud_functions && chown -R node:node /app/cloud_functions ${ENV.CUSTOM_DEPENDENCY_BASE_PATH}
+
+USER node
+RUN npm install ${ENV.DEPENDENCIES} || true
+
+RUN curl -o /tmp/cloud_functions.tar https://${ENV.RUNTIME_DOMAIN}/_/${process.env.SERVER_SECRET}/cloud_functions/tar && tar -xf /tmp/cloud_functions.tar -C /app/cloud_functions && rm /tmp/cloud_functions.tar
+
+FROM node:20.10.0
+USER root
+
+EXPOSE 8000
+EXPOSE 9000
+
+USER node
+
+COPY --from=builder ${ENV.CUSTOM_DEPENDENCY_BASE_PATH} ${ENV.CUSTOM_DEPENDENCY_BASE_PATH}
+COPY --from=builder /app /app
+
+RUN ln -s ${ENV.CUSTOM_DEPENDENCY_BASE_PATH} /app/functions/node_modules
+
+ENV ${envVariablesString}
+
+WORKDIR /app
+
+CMD node $FLAGS --experimental-vm-modules --experimental-fetch ./dist/index.js
+`
+    // 设置响应头，返回纯文本内容
+    res.writeHead(200, {
+      'Content-Type': 'text/plain',
+      'Content-Disposition': 'attachment; filename="Dockerfile"',
+    })
+
+    res.end(DOCKER_FILE)
+  } catch (error) {
+    // Handle errors by sending an internal server error status and the error message
+    res.status(500).send(`Error generating Dockerfile: ${error.message}`)
+  }
+}

runtimes/nodejs/src/handler/router.ts

@@ -13,6 +13,11 @@ import { handlePackageTypings } from './typings'
 import { generateUUID } from '../support/utils'
 import { handleInvokeFunction } from './invoke'
 import { DatabaseAgent } from '../db'
+import Config from '../config'
+import {
+  handleCloudFunctionTarPackage,
+  handleDockerFile,
+} from './docker-file-handler'
 
 /**
  * multer uploader config
@@ -37,7 +42,21 @@ export const router = Router()
 
 router.post('/proxy/:policy', handleDatabaseProxy)
 router.get('/_/typing/package', handlePackageTypings)
+router.get(
+  `/_/${process.env.SERVER_SECRET}/cloud_functions/tar`,
+  handleCloudFunctionTarPackage,
+)
+router.get(
+  `/_/${process.env.SERVER_SECRET}/cloud_functions/dockerfile`,
+  handleDockerFile,
+)
+
 router.get('/_/healthz', (_req, res) => {
+  if (Config.IS_DOCKER_PRODUCT) {
+    res.status(200).send('ok')
+    return
+  }
+
   if (DatabaseAgent.client) {
     res.status(200).send('ok')
   } else {

runtimes/nodejs/src/index.ts

@@ -19,13 +19,13 @@ import xmlparser from 'express-xml-bodyparser'
 
 // init static method of class
 import './support/cloud-sdk'
-import storageServer from './storage-server'
 import { DatabaseChangeStream } from './support/database-change-stream'
 import url from 'url'
 
 import { LspWebSocket } from './support/lsp'
 import { createCloudSdk } from './support/cloud-sdk'
 import { FunctionCache } from './support/engine'
+import storageServer from './storage-server'
 
 require('source-map-support').install({
   emptyCacheBetweenOperations: true,
@@ -38,9 +38,14 @@ globalThis.createCloudSdk = createCloudSdk
 
 const app = express()
 
-DatabaseAgent.ready.then(() => {
-  DatabaseChangeStream.initialize()
-})
+if (Config.IS_DOCKER_PRODUCT) {
+  FunctionCache.initialize()
+  storageServer.close()
+} else {
+  DatabaseAgent.ready.then(() => {
+    DatabaseChangeStream.initialize()
+  })
+}
 
 app.use(
   cors({
@@ -123,9 +128,10 @@ process.on('SIGINT', gracefullyExit)
 
 async function gracefullyExit() {
   await DatabaseAgent.accessor.close()
-  await server.close()
-  await storageServer.close()
-
+  server.close()
+  if (!Config.IS_DOCKER_PRODUCT) {
+    storageServer.close()
+  }
   logger.info('process gracefully exited!')
   process.exit(0)
 }