Improving Self-Hosting and Removing 3rd Party dependencies.

packages/api/package.json

@@ -118,7 +118,10 @@
     "voca": "^1.4.0",
     "winston": "^3.3.3",
     "yaml": "^2.4.1",
-    "youtubei": "^1.5.4"
+    "youtubei": "^1.5.4",
+    "@aws-sdk/client-s3": "^3.679.0",
+    "@aws-sdk/s3-request-presigner": "^3.679.0",
+    "@aws-sdk/lib-storage": "^3.679.0"
   },
   "devDependencies": {
     "@istanbuljs/nyc-config-typescript": "^1.0.2",

packages/api/queue-processor/Dockerfile

@@ -0,0 +1,61 @@
+FROM node:18.16 as builder
+
+WORKDIR /app
+
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
+RUN apt-get update && apt-get install -y g++ make python3
+
+COPY package.json .
+COPY yarn.lock .
+COPY tsconfig.json .
+COPY .prettierrc .
+COPY .eslintrc .
+
+COPY /packages/readabilityjs/package.json ./packages/readabilityjs/package.json
+COPY /packages/api/package.json ./packages/api/package.json
+COPY /packages/text-to-speech/package.json ./packages/text-to-speech/package.json
+COPY /packages/content-handler/package.json ./packages/content-handler/package.json
+COPY /packages/liqe/package.json ./packages/liqe/package.json
+COPY /packages/utils/package.json ./packages/utils/package.json
+
+RUN yarn install --pure-lockfile
+
+ADD /packages/readabilityjs ./packages/readabilityjs
+ADD /packages/api ./packages/api
+ADD /packages/text-to-speech ./packages/text-to-speech
+ADD /packages/content-handler ./packages/content-handler
+ADD /packages/liqe ./packages/liqe
+ADD /packages/utils ./packages/utils
+
+RUN yarn workspace @omnivore/utils build
+RUN yarn workspace @omnivore/text-to-speech-handler build
+RUN yarn workspace @omnivore/content-handler build
+RUN yarn workspace @omnivore/liqe build
+RUN yarn workspace @omnivore/api build
+
+# After building, fetch the production dependencies
+RUN rm -rf /app/packages/api/node_modules
+RUN rm -rf /app/node_modules
+RUN yarn install --pure-lockfile --production
+
+FROM node:18.16 as runner
+LABEL org.opencontainers.image.source="https://github.com/omnivore-app/omnivore"
+
+RUN apt-get update && apt-get install -y netcat-openbsd
+
+WORKDIR /app
+
+ENV NODE_ENV production
+
+COPY --from=builder /app/packages/api/dist /app/packages/api/dist
+COPY --from=builder /app/packages/readabilityjs/ /app/packages/readabilityjs/
+COPY --from=builder /app/packages/api/package.json /app/packages/api/package.json
+COPY --from=builder /app/packages/api/node_modules /app/packages/api/node_modules
+COPY --from=builder /app/node_modules /app/node_modules
+COPY --from=builder /app/package.json /app/package.json
+COPY --from=builder /app/packages/text-to-speech/ /app/packages/text-to-speech/
+COPY --from=builder /app/packages/content-handler/ /app/packages/content-handler/
+COPY --from=builder /app/packages/liqe/ /app/packages/liqe/
+COPY --from=builder /app/packages/utils/ /app/packages/utils/
+
+CMD ["yarn", "workspace", "@omnivore/api", "start_queue_processor"]

packages/api/src/jobs/export.ts

@@ -12,13 +12,24 @@ import { sendExportJobEmail } from '../services/send_emails'
 import { findActiveUser } from '../services/user'
 import { logger } from '../utils/logger'
 import { highlightToMarkdown } from '../utils/parser'
-import { contentFilePath, createGCSFile } from '../utils/uploads'
+import { contentFilePath } from '../utils/uploads'
+import { env } from '../env'
+import { File, Storage } from '@google-cloud/storage'
 
 export interface ExportJobData {
   userId: string
   exportId: string
 }
 
+export const storage = env.fileUpload?.gcsUploadSAKeyFilePath
+  ? new Storage({ keyFilename: env.fileUpload.gcsUploadSAKeyFilePath })
+  : new Storage()
+const bucketName = env.fileUpload.gcsUploadBucket
+
+const createGCSFile = (filename: string): File => {
+  return storage.bucket(bucketName).file(filename)
+}
+
 export const EXPORT_JOB_NAME = 'export'
 
 const itemStateMappping = (state: LibraryItemState) => {

packages/api/src/queue-processor.ts

@@ -56,7 +56,10 @@ import {
   PROCESS_YOUTUBE_VIDEO_JOB_NAME,
 } from './jobs/process-youtube-video'
 import { pruneTrashJob, PRUNE_TRASH_JOB } from './jobs/prune_trash'
-import { refreshAllFeeds } from './jobs/rss/refreshAllFeeds'
+import {
+  REFRESH_ALL_FEEDS_JOB_NAME,
+  refreshAllFeeds,
+} from './jobs/rss/refreshAllFeeds'
 import { refreshFeed } from './jobs/rss/refreshFeed'
 import { savePageJob } from './jobs/save_page'
 import {
@@ -159,25 +162,25 @@ export const createWorker = (connection: ConnectionOptions) =>
     async (job: Job) => {
       const executeJob = async (job: Job) => {
         switch (job.name) {
-          // case 'refresh-all-feeds': {
-          //   const queue = await getQueue()
-          //   const counts = await queue?.getJobCounts('prioritized')
-          //   if (counts && counts.wait > 1000) {
-          //     return
-          //   }
-          //   return await refreshAllFeeds(appDataSource)
-          // }
-          // case 'refresh-feed': {
-          //   return await refreshFeed(job.data)
-          // }
+          case 'refresh-all-feeds': {
+            const queue = await getQueue()
+            const counts = await queue?.getJobCounts('prioritized')
+            if (counts && counts.wait > 1000) {
+              return
+            }
+            return await refreshAllFeeds(appDataSource)
+          }
+          case 'refresh-feed': {
+            return await refreshFeed(job.data)
+          }
           case 'save-page': {
             return savePageJob(job.data, job.attemptsMade)
           }
           // case 'update-pdf-content': {
           //   return updatePDFContentJob(job.data)
           // }
-          // case THUMBNAIL_JOB:
-          //   return findThumbnail(job.data)
+          case THUMBNAIL_JOB:
+            return findThumbnail(job.data)
           case TRIGGER_RULE_JOB_NAME:
             return triggerRule(job.data)
           case UPDATE_LABELS_JOB:
@@ -218,8 +221,8 @@ export const createWorker = (connection: ConnectionOptions) =>
           //   return updateHome(job.data)
           // case SCORE_LIBRARY_ITEM_JOB:
           //   return scoreLibraryItem(job.data)
-          // case GENERATE_PREVIEW_CONTENT_JOB:
-          //   return generatePreviewContent(job.data)
+          case GENERATE_PREVIEW_CONTENT_JOB:
+            return generatePreviewContent(job.data)
           case PRUNE_TRASH_JOB:
             return pruneTrashJob(job.data)
           case EXPIRE_FOLDERS_JOB_NAME:
@@ -260,6 +263,17 @@ const setupCronJobs = async () => {
       },
     }
   )
+
+  await queue.add(
+    REFRESH_ALL_FEEDS_JOB_NAME,
+    {},
+    {
+      priority: getJobPriority(REFRESH_ALL_FEEDS_JOB_NAME),
+      repeat: {
+        every: 14_400_000, // 4 Hours
+      },
+    }
+  )
 }
 
 const main = async () => {

packages/api/src/repository/storage/GcsStorageClient.ts

@@ -0,0 +1,71 @@
+import { SignedUrlParameters, StorageClient, File } from './StorageClient'
+import { Storage, File as GCSFile } from '@google-cloud/storage'
+
+export class GcsStorageClient implements StorageClient {
+  private storage: Storage
+
+  constructor(keyFilename: string | undefined) {
+    this.storage = new Storage({
+      keyFilename,
+    })
+  }
+
+  private convertFileToGeneric(gcsFile: GCSFile): File {
+    return {
+      isPublic: async () => {
+        const [isPublic] = await gcsFile.isPublic()
+        return isPublic
+      },
+      exists: async () => (await gcsFile.exists())[0],
+      download: async () => (await gcsFile.download())[0],
+      bucket: gcsFile.bucket.name,
+      publicUrl: () => gcsFile.publicUrl(),
+      getMetadataMd5: async () => {
+        const [metadata] = await gcsFile.getMetadata()
+        return metadata.md5Hash
+      },
+    }
+  }
+
+  downloadFile(bucket: string, filePath: string): Promise<File> {
+    const file = this.storage.bucket(bucket).file(filePath)
+    return Promise.resolve(this.convertFileToGeneric(file))
+  }
+
+  async getFilesFromPrefix(bucket: string, prefix: string): Promise<File[]> {
+    const [filesWithPrefix] = await this.storage
+      .bucket(bucket)
+      .getFiles({ prefix })
+
+    return filesWithPrefix.map((it: GCSFile) => this.convertFileToGeneric(it))
+  }
+
+  async signedUrl(
+    bucket: string,
+    filePath: string,
+    options: SignedUrlParameters
+  ): Promise<string> {
+    const [url] = await this.storage
+      .bucket(bucket)
+      .file(filePath)
+      .getSignedUrl({ ...options, version: 'v4' })
+
+    return url
+  }
+
+  upload(
+    bucket: string,
+    filePath: string,
+    data: Buffer,
+    options: {
+      contentType?: string
+      public?: boolean
+      timeout?: number
+    }
+  ): Promise<void> {
+    return this.storage
+      .bucket(bucket)
+      .file(filePath)
+      .save(data, { timeout: 30000, ...options })
+  }
+}

packages/api/src/repository/storage/S3StorageClient.ts

@@ -0,0 +1,140 @@
+import { SignedUrlParameters, StorageClient, File } from './StorageClient'
+import {
+  GetObjectCommand,
+  GetObjectCommandOutput,
+  S3Client,
+  ListObjectsV2Command,
+  PutObjectCommand,
+} from '@aws-sdk/client-s3'
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
+import type { Readable } from 'stream'
+
+// While this is listed as S3, for self hosting we will use MinIO, which is
+// S3 Compatible.
+export class S3StorageClient implements StorageClient {
+  private s3Client: S3Client
+  private urlOverride: string | undefined
+
+  constructor(urlOverride: string | undefined) {
+    this.urlOverride = urlOverride
+    this.s3Client = new S3Client({
+      forcePathStyle: true,
+      endpoint: urlOverride,
+    })
+  }
+
+  private convertFileToGeneric(
+    s3File: GetObjectCommandOutput
+  ): Omit<File, 'bucket' | 'publicUrl'> {
+    return {
+      exists: () => {
+        return Promise.resolve(s3File.$metadata.httpStatusCode == 200)
+      },
+      isPublic: async () => Promise.resolve(true),
+      download: async () => this.getFileFromReadable(s3File.Body as Readable),
+      getMetadataMd5: () => Promise.resolve(s3File.ETag),
+    }
+  }
+
+  private getFileFromReadable(stream: Readable): Promise<Buffer> {
+    return new Promise<Buffer>((resolve, reject) => {
+      const chunks: Buffer[] = []
+      stream.on('data', (chunk) => chunks.push(chunk))
+      stream.once('end', () => resolve(Buffer.concat(chunks)))
+      stream.once('error', reject)
+    })
+  }
+
+  async downloadFile(bucket: string, filePath: string): Promise<File> {
+    const s3File = await this.s3Client.send(
+      new GetObjectCommand({
+        Bucket: bucket,
+        Key: filePath, // path to the file you want to download,
+      })
+    )
+
+    return {
+      ...this.convertFileToGeneric(s3File),
+      bucket: bucket,
+      publicUrl: () => `${this.urlOverride ?? ''}/${bucket}/${filePath}`,
+    }
+  }
+
+  async getFilesFromPrefix(bucket: string, prefix: string): Promise<File[]> {
+    const s3PrefixedFiles = await this.s3Client.send(
+      new ListObjectsV2Command({
+        Bucket: bucket,
+        Prefix: prefix, // path to the file you want to download,
+      })
+    )
+
+    const prefixKeys = s3PrefixedFiles.CommonPrefixes || []
+
+    return prefixKeys
+      .map(({ Prefix }) => Prefix)
+      .map((key) => {
+        return {
+          exists: () => Promise.resolve(true),
+          isPublic: async () => Promise.resolve(true),
+          download: async () => {
+            const s3File = await this.s3Client.send(
+              new GetObjectCommand({
+                Bucket: bucket,
+                Key: key, // path to the file you want to download,
+              })
+            )
+
+            return this.getFileFromReadable(s3File.Body as Readable)
+          },
+          getMetadataMd5: () => Promise.resolve(key),
+          bucket: bucket,
+          publicUrl: () => `${this.urlOverride ?? ''}/${bucket}/${key}`,
+        }
+      })
+  }
+
+  async signedUrl(
+    bucket: string,
+    filePath: string,
+    options: SignedUrlParameters
+  ): Promise<string> {
+    const command =
+      options.action == 'read'
+        ? new GetObjectCommand({
+            Bucket: bucket,
+            Key: filePath, // path to the file you want to download,
+          })
+        : new PutObjectCommand({
+            Bucket: bucket,
+            Key: filePath, // path to the file you want to download,
+          })
+
+    // eslint-disable-next-line @typescript-eslint/no-unsafe-call
+    const url = await getSignedUrl(this.s3Client, command, {
+      expiresIn: 900,
+    })
+
+    return url
+  }
+
+  upload(
+    bucket: string,
+    filePath: string,
+    data: Buffer,
+    options: {
+      contentType?: string
+      public?: boolean
+      timeout?: number
+    }
+  ): Promise<void> {
+    return this.s3Client
+      .send(
+        new PutObjectCommand({
+          Bucket: bucket,
+          Key: filePath,
+          Body: data,
+        })
+      )
+      .then(() => {})
+  }
+}