checked repo

paultltc · paultltc · commit 21768e6dbfce · 2025-03-20T21:59:12.000+01:00
diff --git a/README.md b/README.md
@@ -17,51 +17,13 @@ A scalable multimodal pipeline for processing, indexing, and querying multimodal
 Ever needed to take 8000 PDFs, 2000 videos, and 500 spreadsheets and feed them to an LLM as a knowledge base?
 Well, MMORE is here to help you!
 
-## MMORE Installation Guide
+## :bulb: Quickstart
 
-### Installation Option 1: pip (recommended)
+### Installation
 
-To install all dependencies, run:
+#### (Step 0 – Install system dependencies)
 
-```bash
-pip install -e '.[all]'
-```
-
-To install only processor-related dependencies, run:
-
-```bash
-pip install -e '.[processor]'
-```
-
-To install only RAG-related dependencies, run:
-
-```bash
-pip install -e '.[rag]'
-```
-
----
-
-
-### Minimal Example
-
-```python
-from mmore.process.processors.pdf_processor import PDFProcessor 
-from mmore.process.processors.base import ProcessorConfig
-from mmore.type import MultimodalSample
-
-pdf_file_paths = ["examples/sample_data/pdf/calendar.pdf"]
-out_file = "results/example.jsonl"
-
-pdf_processor_config = ProcessorConfig(custom_config={"output_path": "results"})
-pdf_processor = PDFProcessor(config=pdf_processor_config)
-result_pdf = pdf_processor.process_batch(pdf_file_paths, True, 1) # args: file_paths, fast mode (True/False), num_workers
-
-MultimodalSample.to_jsonl(out_file, result_pdf)
-```
-
-### Installation Option 2: uv
-
-#### Step 1: Install system dependencies
+Our package requires system dependencies. This snippet will take care of installing them!
 
 ```bash
 sudo apt update
@@ -70,95 +32,52 @@ sudo apt install -y ffmpeg libsm6 libxext6 chromium-browser libnss3 \
   libxext6 libxfixes3 libxrender1 libasound2 libatk1.0-0 libgtk-3-0 libreoffice
 ```
 
-#### Step 2: Install `uv`
-
-Refer to the [uv installation guide](https://docs.astral.sh/uv/getting-started/installation/) for detailed instructions.
-```
-curl -LsSf https://astral.sh/uv/install.sh | sh
-```
-
-#### Step 3: Clone this repository
+#### Step 1 – Install MMORE
 
-```bash
-git clone https://github.com/swiss-ai/mmore
-cd mmore
-```
-
-#### Step 4: Install project and dependencies
+To install the package simply run:
 
 ```bash
-uv sync
+pip install -e .
 ```
 
-For CPU-only installation, use:
+To install additional RAG-related dependencies, run:
 
 ```bash
-uv sync --extra cpu
+pip install -e '.[rag]'
 ```
 
-#### Step 5: Run a test command
-
-Activate the virtual environment before running commands:
+> :warning: This is a big package with a lot of dependencies, so we recommend to use `uv` to handle `pip` installations. [Check our tutorial on uv](./docs/uv.md).
 
-```bash
-python -m venv .venv
-source .venv/bin/activate
-```
+### Minimal Example
 
-Alternatively, prepend each command with `uv run`:
+You can use our predefined CLI commands to execute parts of the pipeline. Note that you might need to prepend `python -m` to the command if the package does not properly create bash aliases.
 
 ```bash
 # Run processing
-python -m mmore process --config-file examples/process/config.yaml
+mmore process --config-file examples/process/config.yaml
 
 # Run indexer
-python -m mmore index --config-file examples/index/config.yaml
+mmore index --config-file examples/index/config.yaml
 
 # Run RAG
-python -m mmore rag --config-file examples/rag/api/rag_api.yaml
-```
-
----
-
-### Installation Option 3: Docker
-
-**Note:** For manual installation without Docker, refer to the section below.
-
-#### Step 1: Install Docker
-
-Follow the official [Docker installation guide](https://docs.docker.com/get-started/get-docker/).
-
-#### Step 2: Build the Docker image
-
-```bash
-docker build . --tag mmore
-```
-
-To build for CPU-only platforms (results in a smaller image size):
-
-```bash
-docker build --build-arg PLATFORM=cpu -t mmore .
-```
-
-#### Step 3: Start an interactive session
-
-```bash
-docker run -it -v ./test_data:/app/test_data mmore
+mmore rag --config-file examples/rag/api/rag_api.yaml
 ```
 
-*Note:* The `test_data` folder is mapped to `/app/test_data` inside the container, corresponding to the default path in `examples/process_config.yaml`.
+You can also use our package in python code as shown here:
 
-#### Step 4: Run the application inside the container
+```python
+from mmore.process.processors.pdf_processor import PDFProcessor 
+from mmore.process.processors.base import ProcessorConfig
+from mmore.type import MultimodalSample
 
-```bash
-# Run processing
-mmore process --config-file examples/process/config.yaml
+pdf_file_paths = ["examples/sample_data/pdf/calendar.pdf"]
+out_file = "results/example.jsonl"
 
-# Run indexer
-mmore index --config-file examples/index/config.yaml
+pdf_processor_config = ProcessorConfig(custom_config={"output_path": "results"})
+pdf_processor = PDFProcessor(config=pdf_processor_config)
+result_pdf = pdf_processor.process_batch(pdf_file_paths, True, 1) # args: file_paths, fast mode (True/False), num_workers
 
-# Run RAG
-mmore rag --config-file examples/rag/api/rag_api.yaml
+MultimodalSample.to_jsonl(out_file, result_pdf)
 ```
 
 ---
diff --git a/docs/docker.md b/docs/docker.md
@@ -0,0 +1,27 @@
+### Installation Option 3: Docker
+
+**Note:** For manual installation without Docker, refer to the section below.
+
+#### Step 1: Install Docker
+
+Follow the official [Docker installation guide](https://docs.docker.com/get-started/get-docker/).
+
+#### Step 2: Build the Docker image
+
+```bash
+docker build . --tag mmore
+```
+
+To build for CPU-only platforms (results in a smaller image size):
+
+```bash
+docker build --build-arg PLATFORM=cpu -t mmore .
+```
+
+#### Step 3: Start an interactive session
+
+```bash
+docker run -it -v ./test_data:/app/test_data mmore
+```
+
+*Note:* The `test_data` folder is mapped to `/app/test_data` inside the container, corresponding to the default path in `examples/process_config.yaml`.
diff --git a/examples/index/config.yaml b/examples/index/config.yaml
@@ -1,11 +1,3 @@
-<<<<<<< HEAD
-dense_model:
-  model_name: BAAI/bge-small-en
-sparse_model:
-  model_name: splade
-db:
-  uri: ./examples/index/sample.db
-=======
 indexer:
   dense_model:
     model_name: sentence-transformers/all-MiniLM-L6-v2
@@ -18,4 +10,3 @@ indexer:
     name: my_db
 collection_name: my_docs
 documents_path: 'examples/process/outputs/merged/merged_results.jsonl'
->>>>>>> 21388c46a4f8f731769264e8ef0d85d055f7a6d8
diff --git a/examples/rag/config_api.yaml b/examples/rag/config_api.yaml
@@ -8,16 +8,6 @@ rag:
   # Retriever Config
   retriever:
     db:
-<<<<<<< HEAD:examples/rag/config_api.yaml
-      uri: ./examples/rag/ner.db
-    hybrid_search_weight: 0.5
-    k: 5
-  system_prompt: "Use the following context to answer the questions.\n\nContext:\n{context}"
-api:    
-    endpoint: '/rag'
-    port: 8000
-    host: 'localhost'
-=======
       uri: ./proc_demo.db
       name: my_db
     hybrid_search_weight: 0.5
@@ -30,4 +20,3 @@ mode_args:
   endpoint: '/rag'
   port: 8000
   host: 'localhost'
->>>>>>> 21388c46a4f8f731769264e8ef0d85d055f7a6d8:examples/rag/api/rag_api.yaml
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,3 +1,7 @@
+[build-system]
+requires = ["setuptools", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
 [project]
 name = "mmore"
 version = "1.0.0"
@@ -19,23 +23,10 @@ dependencies = [
     "python-dotenv==1.0.1",
     "dacite==1.8.1",
     "click>=8.1.7",
-    "dask-cuda>=24.10.0",
-    "cuda-python>=12.6.2",
-    "ucx-py-cu12>=0.40.0",
+    "dask[distributed]>=2025.2.0",
     "pytest>=8.3.4",
     "validators==0.34.0",
     "httpx==0.27.2",
-]
-
-[project.optional-dependencies]
-cpu = [
-    "torch>=2.5.1",
-]
-cu124 = [
-    "torch>=2.5.1",
-]
-
-process = [
     "Pillow",
     "PyMuPDF",
     "Unidecode",
@@ -69,36 +60,16 @@ process = [
     "pymongo==4.9.2"
 ]
 
-rag = [
-  "accelerate",
-  "langchain-anthropic==0.3.4",
-  "langchain-aws",
-  "langchain-cohere==0.4.2",
-  "langchain-huggingface==0.1.2",
-  "langchain-milvus==0.1.8",
-  "langchain-mistralai==0.2.7",
-  "langchain-nvidia-ai-endpoints",
-  "langchain-openai==0.3.7",
-  "langchain==0.3.20",
-  "langdetect>=1.0.9",
-  "langserve[all]==0.3.1",
-  "pymilvus[model]==2.5.0",
-  "ragas==0.2.6",
-  "nltk>=3.9",
+[project.optional-dependencies]
+cpu = [
+    "torch>=2.5.1",
+]
+cu124 = [
+    "torch>=2.5.1",
 ]
 
-all = [
+rag = [
   "accelerate",
-  "Pillow",
-  "PyMuPDF",
-  "Unidecode",
-  "bokeh",
-  "chonkie==0.2.1.post1",
-  "clean-text",
-  "datatrove[processing]",
-  "docx2pdf",
-  "fastapi==0.115.5",
-  "fastapi[standard]",
   "langchain-anthropic==0.3.4",
   "langchain-aws",
   "langchain-cohere==0.4.2",
@@ -110,28 +81,9 @@ all = [
   "langchain==0.3.20",
   "langdetect>=1.0.9",
   "langserve[all]==0.3.1",
-  "lxml_html_clean",
-  "markdown==3.7",
-  "markdownify==0.13.1",
-  "marker-pdf==1.6.0",
-  "motor==3.6.0",
-  "moviepy==2.1.1",
-  "nltk>=3.9",
-  "openpyxl==3.1.5",
-  "py7zr==0.22.0",
-  "pydantic== 2.10.4",
   "pymilvus[model]==2.5.0",
-  "pymongo==4.9.2",
-  "python-docx",
-  "python-pptx",
   "ragas==0.2.6",
-  "rarfile==4.2",
-  "requests==2.28.1",
-  "selenium==4.27.1",
-  "surya-ocr>=0.8.3",
-  "trafilatura==1.4.0",
-  "validators==0.34.0",
-  "xlrd==2.0.1",
+  "nltk>=3.9",
 ]
 
 [tool.uv]
@@ -160,8 +112,4 @@ url = "https://download.pytorch.org/whl/cu124"
 explicit = true
 
 [project.scripts]
-mmore = "mmore:__main__"
-
-[build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"
+mmore = "mmore:__main__"
diff --git a/requirements.txt b/requirements.txt
