Korvus x Firecrawl

Author: SilasMarvin

.env.development

@@ -0,0 +1,4 @@
+CRAWL_URL="postgresml.org"
+CRAWL_LIMIT=100
+KORVUS_DATABASE_URL=""
+FIRECRAWL_API_KEY=""

.gitignore

@@ -0,0 +1,2 @@
+.env
+venv

LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2024 PostgresML Team
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

@@ -0,0 +1,68 @@
+# Korvus x Firecrawl Example
+
+This example application demonstrates how to perform web crawling, semantic search, and Retrieval-Augmented Generation (RAG) using [Korvus](https://github.com/postgresml/korvus) and [Firecrawl](https://www.firecrawl.dev/).
+
+## Features
+
+- Web crawling using Firecrawl
+- Semantic search over crawled content
+- RAG (Retrieval-Augmented Generation) for question answering
+
+## Prerequisites
+
+- Python 3.7+
+- Firecrawl API key
+- PostgresML database URL
+
+## Installation
+
+1. Clone this repository:
+   ```
+   git clone https://github.com/yourusername/korvus-firecrawl-example.git
+   cd korvus-firecrawl-example
+   ```
+
+2. Install the required packages:
+   ```
+   pip install korvus firecrawl python-dotenv rich
+   ```
+
+3. Create a `.env` file in the project root and add your credentials:
+   ```
+   FIRECRAWL_API_KEY=your_firecrawl_api_key
+   KORVUS_DATABASE_URL=your_postgresml_database_url
+   CRAWL_URL=https://example.com
+   CRAWL_LIMIT=100
+   ```
+
+## Usage
+
+The application supports three main actions: crawl, search, and rag.
+
+1. Crawl a website:
+   ```
+   python main.py crawl
+   ```
+
+2. Perform semantic search:
+   ```
+   python main.py search
+   ```
+
+3. Use RAG for question answering:
+   ```
+   python main.py rag
+   ```
+
+For search and RAG, you'll be prompted to enter queries. Type 'q' to quit the input loop.
+
+## How it works
+
+1. The application uses Firecrawl to crawl the specified website and extract markdown content.
+2. Crawled data is processed and stored using Korvus.
+3. Semantic search allows you to find relevant documents based on your queries.
+4. RAG combines retrieved context with a language model to answer questions.
+
+## Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.

main.py

@@ -0,0 +1,167 @@
+from korvus import Collection, Pipeline
+from firecrawl import FirecrawlApp
+import os
+import time
+import asyncio
+from rich import print
+from rich.pretty import pprint
+from dotenv import load_dotenv
+import argparse
+
+
+# Load variables from our .env file
+load_dotenv()
+
+
+# Configure our program args
+parser = argparse.ArgumentParser(description="Example Korvus x Firecrawl")
+parser.add_argument(
+    "action", choices=["crawl", "search", "rag"], help="Action to perform"
+)
+
+
+# Initialize the FirecrawlApp with your API key
+firecrawl = FirecrawlApp(api_key=os.environ["FIRECRAWL_API_KEY"])
+
+
+# Define our Pipeline and Collection
+pipeline = Pipeline(
+    "v0",
+    {
+        "markdown": {
+            "splitter": {"model": "markdown"},
+            "semantic_search": {
+                "model": "mixedbread-ai/mxbai-embed-large-v1",
+            },
+        },
+    },
+)
+collection = Collection(
+    "korvus-firecrawl-example-0", database_url=os.environ["KORVUS_DATABASE_URL"]
+)
+
+
+# Add our Pipeline to our Collection
+async def add_pipeline():
+    await collection.add_pipeline(pipeline)
+
+
+# Crawl with Firecrawl
+def crawl():
+    print("Crawling...")
+    job = firecrawl.crawl_url(
+        os.environ["CRAWL_URL"],
+        params={
+            "limit": int(os.environ["CRAWL_LIMIT"]),
+            "scrapeOptions": {"formats": ["markdown"]},
+        },
+        poll_interval=30,
+    )
+    return job
+
+
+# Do RAG
+async def do_rag(user_query):
+    results = await collection.rag(
+        {
+            "CONTEXT": {
+                "vector_search": {
+                    "query": {
+                        "fields": {
+                            "markdown": {
+                                "query": user_query,
+                                "parameters": {
+                                    "prompt": "Represent this sentence for searching relevant passages: "
+                                },
+                            }
+                        },
+                    },
+                    "document": {"keys": ["id"]},
+                    "rerank": {
+                        "model": "mixedbread-ai/mxbai-rerank-base-v1",
+                        "query": user_query,
+                        "num_documents_to_rerank": 100,
+                    },
+                    "limit": 5,
+                },
+                "aggregate": {"join": "\n\n\n"},
+            },
+            "chat": {
+                "model": "meta-llama/Meta-Llama-3.1-405B-Instruct",
+                "messages": [
+                    {
+                        "role": "system",
+                        "content": "You are a question and answering bot. Answer the users question given the context succinctly.",
+                    },
+                    {
+                        "role": "user",
+                        "content": f"Given the context\n\n:{{CONTEXT}}\n\nAnswer the question: {user_query}",
+                    },
+                ],
+                "max_tokens": 256,
+            },
+        },
+        pipeline,
+    )
+    return results
+
+
+# Do search
+async def do_search(user_query):
+    results = await collection.search(
+        {
+            "query": {
+                "semantic_search": {
+                    "markdown": {
+                        "query": user_query,
+                    },
+                },
+            },
+            "limit": 5,
+        },
+        pipeline,
+    )
+    return results
+
+
+# Get user input and call our callback
+async def input_loop(callback):
+    while True:
+        query = input("Enter your query (or 'q' to quit): ")
+        if query.lower() == "q":
+            break
+        results = await callback(query)
+        print("\n[bold]Results:[/bold]\n")
+        pprint(results, max_length=2, max_string=100)
+
+
+# Our main function
+async def main():
+    args = parser.parse_args()
+
+    if args.action == "crawl":
+        # Add our Pipeline to our Collection
+        # We only ever need to do this once
+        # Calling it more than once does nothing
+        await add_pipeline()
+
+        # Crawl the website
+        results = crawl()
+
+        # Construct our documents to upsert
+        documents = [
+            {"id": data["metadata"]["sourceURL"], "markdown": data["markdown"]}
+            for data in results["data"]
+        ]
+
+        # Upsert our documents
+        await collection.upsert_documents(documents)
+
+    elif args.action == "rag":
+        await input_loop(do_rag)
+
+    elif args.action == "search":
+        await input_loop(do_search)
+
+
+asyncio.run(main())