Improving Self-Hosting and Removing 3rd Party dependencies.

README.md

@@ -151,24 +151,7 @@ is done fetching your content you will see it in your library.
 
 ## How to deploy to your own server
 
-Omnivore was originally designed to be deployed on GCP and takes advantage
-of some of GCP's PaaS features. We are working to make Omnivore more portable
-so you can easily run the service on your own infrastructure. You can track
-progress here: <https://github.com/omnivore-app/omnivore/issues/25>
-
-To deploy Omnivore on your own hardware you will need to deploy three
-dockerized services and configure access to a postgres service. To handle
-PDF documents you will need to configure access to a Google Cloud Storage
-bucket.
-
-- `packages/api` - the backend API service
-- `packages/web` - the web frontend (can easily be deployed to vercel)
-- `packages/puppeteer-parse` - the content fetching service (can easily
-  be deployed as an AWS lambda or GCP Cloud Function)
-
-Additionally, you will need to run our database migrations to initialize
-your database. These are dockerized and can be run with the
-`packages/db` service.
+A guide for running a self hosted server can be found [here](./self-hosting/GUIDE.md)
 
 ## License
 

imageproxy/Dockerfile

@@ -1,4 +1,4 @@
-FROM willnorris/imageproxy:v0.10.0 as build
+FROM ghcr.io/willnorris/imageproxy:main as build
 
 # Above imageproxy image is built from scratch image and is barebones
 # Switching over to ubuntu base image to allow us to debug better.

packages/api/package.json

@@ -120,7 +120,10 @@
     "voca": "^1.4.0",
     "winston": "^3.3.3",
     "yaml": "^2.4.1",
-    "youtubei": "^1.5.4"
+    "youtubei": "^1.5.4",
+    "@aws-sdk/client-s3": "^3.679.0",
+    "@aws-sdk/s3-request-presigner": "^3.679.0",
+    "@aws-sdk/lib-storage": "^3.679.0"
   },
   "devDependencies": {
     "@istanbuljs/nyc-config-typescript": "^1.0.2",

packages/api/queue-processor/Dockerfile

@@ -0,0 +1,61 @@
+FROM node:18.16 as builder
+
+WORKDIR /app
+
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true
+RUN apt-get update && apt-get install -y g++ make python3
+
+COPY package.json .
+COPY yarn.lock .
+COPY tsconfig.json .
+COPY .prettierrc .
+COPY .eslintrc .
+
+COPY /packages/readabilityjs/package.json ./packages/readabilityjs/package.json
+COPY /packages/api/package.json ./packages/api/package.json
+COPY /packages/text-to-speech/package.json ./packages/text-to-speech/package.json
+COPY /packages/content-handler/package.json ./packages/content-handler/package.json
+COPY /packages/liqe/package.json ./packages/liqe/package.json
+COPY /packages/utils/package.json ./packages/utils/package.json
+
+RUN yarn install --pure-lockfile
+
+ADD /packages/readabilityjs ./packages/readabilityjs
+ADD /packages/api ./packages/api
+ADD /packages/text-to-speech ./packages/text-to-speech
+ADD /packages/content-handler ./packages/content-handler
+ADD /packages/liqe ./packages/liqe
+ADD /packages/utils ./packages/utils
+
+RUN yarn workspace @omnivore/utils build
+RUN yarn workspace @omnivore/text-to-speech-handler build
+RUN yarn workspace @omnivore/content-handler build
+RUN yarn workspace @omnivore/liqe build
+RUN yarn workspace @omnivore/api build
+
+# After building, fetch the production dependencies
+RUN rm -rf /app/packages/api/node_modules
+RUN rm -rf /app/node_modules
+RUN yarn install --pure-lockfile --production
+
+FROM node:18.16 as runner
+LABEL org.opencontainers.image.source="https://github.com/omnivore-app/omnivore"
+
+RUN apt-get update && apt-get install -y netcat-openbsd
+
+WORKDIR /app
+
+ENV NODE_ENV production
+
+COPY --from=builder /app/packages/api/dist /app/packages/api/dist
+COPY --from=builder /app/packages/readabilityjs/ /app/packages/readabilityjs/
+COPY --from=builder /app/packages/api/package.json /app/packages/api/package.json
+COPY --from=builder /app/packages/api/node_modules /app/packages/api/node_modules
+COPY --from=builder /app/node_modules /app/node_modules
+COPY --from=builder /app/package.json /app/package.json
+COPY --from=builder /app/packages/text-to-speech/ /app/packages/text-to-speech/
+COPY --from=builder /app/packages/content-handler/ /app/packages/content-handler/
+COPY --from=builder /app/packages/liqe/ /app/packages/liqe/
+COPY --from=builder /app/packages/utils/ /app/packages/utils/
+
+CMD ["yarn", "workspace", "@omnivore/api", "start_queue_processor"]

packages/api/src/jobs/export.ts

@@ -1,10 +1,6 @@
 import archiver, { Archiver } from 'archiver'
 import { v4 as uuidv4 } from 'uuid'
-import {
-  ContentReaderType,
-  LibraryItem,
-  LibraryItemState,
-} from '../entity/library_item'
+import { LibraryItem, LibraryItemState } from '../entity/library_item'
 import { TaskState } from '../generated/graphql'
 import { findExportById, saveExport } from '../services/export'
 import { findHighlightsByLibraryItemId } from '../services/highlights'
@@ -17,12 +13,11 @@ import { sendExportJobEmail } from '../services/send_emails'
 import { findActiveUser } from '../services/user'
 import { logger } from '../utils/logger'
 import { highlightToMarkdown } from '../utils/parser'
-import {
-  contentFilePath,
-  createGCSFile,
-  generateUploadFilePathName,
-} from '../utils/uploads'
-import { batch } from 'googleapis/build/src/apis/batch'
+import { env } from '../env'
+import { storage } from '../repository/storage/storage'
+import { File } from '../repository/storage/StorageClient'
+import { Readable } from 'stream'
+import { contentFilePath, generateUploadFilePathName } from '../utils/uploads'
 import { getRepository } from '../repository'
 import { UploadFile } from '../entity/upload_file'
 
@@ -31,6 +26,12 @@ export interface ExportJobData {
   exportId: string
 }
 
+const bucketName = env.fileUpload.gcsUploadBucket
+
+const createGCSFile = (filename: string): File => {
+  return storage.createFile(bucketName, filename)
+}
+
 export const EXPORT_JOB_NAME = 'export'
 
 const itemStateMappping = (state: LibraryItemState) => {
@@ -61,7 +62,7 @@ const uploadContent = async (
   const file = createGCSFile(filePath)
 
   // check if file is already uploaded
-  const [exists] = await file.exists()
+  const exists = await file.exists()
   if (!exists) {
     logger.info(`File not found: ${filePath}`)
 
@@ -81,10 +82,14 @@ const uploadContent = async (
       contentType: 'text/html',
       private: true,
     })
+    archive.append(Readable.from(item.readableContent), {
+      name: `content/${libraryItem.slug}.html`,
+    })
   }
 
   // append the existing file to the archive
-  archive.append(file.createReadStream(), {
+  const content = await file.download()
+  archive.append(Readable.from(content.toString()), {
     name: `content/${libraryItem.slug}.html`,
   })
 }
@@ -97,17 +102,19 @@ const uploadPdfContent = async (
     id: libraryItem.uploadFileId,
   })
   if (!upload || !upload.fileName) {
-    console.log(`upload does not have a filename: ${upload}`)
+    console.log(
+      `upload does not have a filename: ${upload?.fileName ?? 'empty'}`
+    )
     return
   }
 
   const filePath = generateUploadFilePathName(upload.id, upload.fileName)
   const file = createGCSFile(filePath)
-  const [exists] = await file.exists()
+  const exists = await file.exists()
   if (exists) {
     console.log(`adding PDF file: ${filePath}`)
     // append the existing file to the archive
-    archive.append(file.createReadStream(), {
+    archive.append(await file.download(), {
       name: `content/${libraryItem.slug}.pdf`,
     })
   }
@@ -238,20 +245,25 @@ export const exportJob = async (jobData: ExportJobData) => {
 
     // Create a write stream
     const writeStream = file.createWriteStream({
-      metadata: {
-        contentType: 'application/zip',
-      },
+      contentType: 'application/zip',
+    })
+
+    const finishedPromise = new Promise<void>((resolve, reject) => {
+      if (writeStream.closed) {
+        resolve()
+      }
+      writeStream.on('finish', () => {
+        logger.info('File successfully written to GCS')
+        resolve()
+      })
+      writeStream.on('error', reject)
     })
 
     // Handle any errors in the streams
     writeStream.on('error', (err) => {
       logger.error('Error writing to GCS:', err)
     })
 
-    writeStream.on('finish', () => {
-      logger.info('File successfully written to GCS')
-    })
-
     // Initialize archiver for zipping files
     const archive = archiver('zip', {
       zlib: { level: 9 }, // Compression level
@@ -264,7 +276,6 @@ export const exportJob = async (jobData: ExportJobData) => {
 
     // Pipe the archiver output to the write stream
     archive.pipe(writeStream)
-
     let cursor = 0
     try {
       // fetch data from the database
@@ -305,17 +316,14 @@ export const exportJob = async (jobData: ExportJobData) => {
     }
 
     // Ensure that the writeStream has finished
-    await new Promise((resolve, reject) => {
-      writeStream.on('finish', resolve)
-      writeStream.on('error', reject)
-    })
+    await finishedPromise
 
     logger.info(`export completed, exported ${cursor} items`, {
       userId,
     })
 
     // generate a temporary signed url for the zip file
-    const [signedUrl] = await file.getSignedUrl({
+    const signedUrl = await storage.signedUrl(bucketName, fullPath, {
       action: 'read',
       expires: Date.now() + 168 * 60 * 60 * 1000, // one week
     })

packages/api/src/queue-processor.ts

@@ -56,7 +56,10 @@ import {
   PROCESS_YOUTUBE_VIDEO_JOB_NAME,
 } from './jobs/process-youtube-video'
 import { pruneTrashJob, PRUNE_TRASH_JOB } from './jobs/prune_trash'
-import { refreshAllFeeds } from './jobs/rss/refreshAllFeeds'
+import {
+  REFRESH_ALL_FEEDS_JOB_NAME,
+  refreshAllFeeds,
+} from './jobs/rss/refreshAllFeeds'
 import { refreshFeed } from './jobs/rss/refreshFeed'
 import { savePageJob } from './jobs/save_page'
 import {
@@ -159,25 +162,25 @@ export const createWorker = (connection: ConnectionOptions) =>
     async (job: Job) => {
       const executeJob = async (job: Job) => {
         switch (job.name) {
-          // case 'refresh-all-feeds': {
-          //   const queue = await getQueue()
-          //   const counts = await queue?.getJobCounts('prioritized')
-          //   if (counts && counts.wait > 1000) {
-          //     return
-          //   }
-          //   return await refreshAllFeeds(appDataSource)
-          // }
-          // case 'refresh-feed': {
-          //   return await refreshFeed(job.data)
-          // }
+          case 'refresh-all-feeds': {
+            const queue = await getQueue()
+            const counts = await queue?.getJobCounts('prioritized')
+            if (counts && counts.wait > 1000) {
+              return
+            }
+            return await refreshAllFeeds(appDataSource)
+          }
+          case 'refresh-feed': {
+            return await refreshFeed(job.data)
+          }
           case 'save-page': {
             return savePageJob(job.data, job.attemptsMade)
           }
           // case 'update-pdf-content': {
           //   return updatePDFContentJob(job.data)
           // }
-          // case THUMBNAIL_JOB:
-          //   return findThumbnail(job.data)
+          case THUMBNAIL_JOB:
+            return findThumbnail(job.data)
           case TRIGGER_RULE_JOB_NAME:
             return triggerRule(job.data)
           case UPDATE_LABELS_JOB:
@@ -218,8 +221,8 @@ export const createWorker = (connection: ConnectionOptions) =>
           //   return updateHome(job.data)
           // case SCORE_LIBRARY_ITEM_JOB:
           //   return scoreLibraryItem(job.data)
-          // case GENERATE_PREVIEW_CONTENT_JOB:
-          //   return generatePreviewContent(job.data)
+          case GENERATE_PREVIEW_CONTENT_JOB:
+            return generatePreviewContent(job.data)
           case PRUNE_TRASH_JOB:
             return pruneTrashJob(job.data)
           case EXPIRE_FOLDERS_JOB_NAME:
@@ -260,6 +263,17 @@ const setupCronJobs = async () => {
       },
     }
   )
+
+  await queue.add(
+    REFRESH_ALL_FEEDS_JOB_NAME,
+    {},
+    {
+      priority: getJobPriority(REFRESH_ALL_FEEDS_JOB_NAME),
+      repeat: {
+        every: 14_400_000, // 4 Hours
+      },
+    }
+  )
 }
 
 const main = async () => {