Merge pull request #2 from papermerge/generate-doc-previews-after-ocr

generate document previews after OCR

Author: ciur

README.md

@@ -1 +1,45 @@
 # OCR Worker
+
+Performs OCR on the documents. Optionally can download/upload documents
+from S3 storage.
+
+## Run it:
+
+    poetry run task worker
+
+## Configuration
+
+OCR Worker is configured via environment variables
+
+### PAPERMERGE__DATABASE__URL
+
+Database URL (URI). For details see: [Database URLs](https://docs.sqlalchemy.org/en/20/core/engines.html#database-urls)
+Default value is `sqlite:////db/db.sqlite3`
+
+Example:
+
+    export PAPERMERGE__DATABASE__URL=sqlite:////opt/cocodb.sqlite3
+
+### PAPERMERGE__REDIS__URL
+
+Redis URL (URI).
+If no value is provided, then it will not connect to Redis.
+
+Example:
+
+    export PAPERMERGE__REDIS__URL=redis://localhost:6379/0
+
+### PAPERMERGE__MAIN__LOGGING_CFG
+
+Example:
+
+    export PAPERMERGE__MAIN__LOGGING_CFG=/etc/papermerge/logging.yaml
+
+### PAPERMERGE__MAIN__MEDIA_ROOT
+
+Path to media root. If no value provided, current working directory
+is used as media root.
+
+Example:
+
+    export PAPERMERGE__MAIN__MEDIA_ROOT=/opt/media_root

ocrworker/celery_app.py

@@ -1,9 +1,11 @@
+import logging
 from celery import Celery
 from ocrworker import config, utils
 from celery.signals import setup_logging
 
 
 settings = config.get_settings()
+logger = logging.getLogger(__name__)
 
 app = Celery(
     "ocrworker",

ocrworker/constants.py

@@ -8,3 +8,4 @@
 PAGE_PDF = "page.pdf"
 INDEX_ADD_DOCS = "index_add_docs"
 WORKER_OCR_DOCUMENT = "worker_ocr_document"
+S3_WORKER_GENERATE_PREVIEW = "s3_worker_generate_preview"

ocrworker/tasks.py

@@ -80,6 +80,7 @@ def ocr_document_task(document_id: str, lang: str):
             target_docver_id=target_docver_uuid,
             target_page_ids=target_page_uuids,
         ).set(queue=prefixed(const.OCR))
+        | generate_preview.s(doc_id=document_id).set(queue=prefixed(const.OCR))
         | notify_index_task.s(doc_id=document_id).set(queue=prefixed(const.OCR))
     )
     # I've tried workflow.apply_async(queue=prefixed(OCR))
@@ -198,6 +199,19 @@ def notify_index_task(_, **kwargs):
     )
 
 
+@shared_task()
+def generate_preview(_, **kwargs):
+    logger.debug(f"Generate thumbnail/page previews for doc_id={kwargs}")
+
+    doc_id = kwargs["doc_id"]
+
+    celery_app.send_task(
+        const.S3_WORKER_GENERATE_PREVIEW,
+        kwargs={"doc_id": doc_id},
+        route_name="s3preview",
+    )
+
+
 def prefixed(name: str) -> str:
     pref = settings.papermerge__main__prefix
     if pref:

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ocrworker"
-version = "0.1.0"
+version = "0.2.0"
 description = "OCR Worker"
 authors = ["Eugen Ciur <[email protected]>"]
 readme = "README.md"
@@ -32,7 +32,7 @@ databases = ["mysqlclient", "psycopg2"]
 ocr = "ocrworker.cli.ocr:app"
 
 [tool.taskipy.tasks]
-worker = "celery -A ocrworker.celery_app worker -E -c 8 -Q dev-coco_ocr"
+worker = "celery -A ocrworker.celery_app worker -E -c 8 -Q ocr"
 
 [tool.poetry.group.dev.dependencies]
 black = "^24.4.2"