feat: route LLM calls through LiteLLM for multi-provider support

Author: prodmanpd

backend/app/condense.py

@@ -184,12 +184,11 @@ def _condense_with_claude(section: dict, max_chars: int) -> Optional[dict]:
         return None
 
     try:
-        from anthropic import Anthropic
+        import litellm
 
         allowed_ids = set(_union_memory_ids(elements))
         by_id = _elements_by_id(elements)
 
-        client = Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"])
         model = os.environ.get(
             "CORTEX_CONDENSE_MODEL",
             os.environ.get("CORTEX_EXTRACTION_MODEL", "claude-opus-4-5"),
@@ -203,14 +202,18 @@ def _condense_with_claude(section: dict, max_chars: int) -> Optional[dict]:
         )
         user = _build_user_prompt(section, elements)
 
-        response = client.messages.create(
+        response = litellm.completion(
             model=model,
             max_tokens=300,
-            system=system,
-            messages=[{"role": "user", "content": user}],
+            messages=[
+                {"role": "system", "content": system},
+                {"role": "user", "content": user},
+            ],
+            # Drop provider-unsupported params so one config works across providers.
+            drop_params=True,
         )
 
-        text = response.content[0].text.strip()
+        text = (response.choices[0].message.content or "").strip()
         text = re.sub(r"^```(?:json)?\s*", "", text)
         text = re.sub(r"\s*```$", "", text)
         data = json.loads(text)

backend/requirements.txt

@@ -3,6 +3,7 @@ uvicorn>=0.20.0
 pydantic>=2.0.0
 python-dotenv>=1.0.0
 anthropic>=0.40.0
+litellm>=1.89.0,<2.0.0
 sqlite-vec>=0.1.9
 argon2-cffi>=23.1.0
 cryptography>=42.0.0

ingest.py

@@ -19,9 +19,11 @@
 from instrumentation import setup_tracing, get_tracer
 setup_tracing(project_name="cortex")
 
-from anthropic import Anthropic
+import litellm
 
-client = Anthropic(api_key=os.environ.get("ANTHROPIC_API_KEY"))
+# Model routed through LiteLLM (Claude by default). Set CORTEX_MODEL to use any
+# other provider LiteLLM supports; credentials come from that provider's env var.
+DEFAULT_MODEL = os.environ.get("CORTEX_MODEL", "claude-opus-4-5")
 _tracer = get_tracer()
 
 
@@ -89,23 +91,27 @@ def extract_context(raw_text: str, source: str = "unknown") -> dict:
     now = datetime.now().isoformat()
     source_id = make_id("src_", source + now[:16])
 
-    response = client.messages.create(
-        model="claude-opus-4-5",
+    response = litellm.completion(
+        model=DEFAULT_MODEL,
         max_tokens=3000,
-        system=EXTRACTION_SYSTEM_PROMPT,
-        messages=[{
-            "role": "user",
-            "content": (
-                f"Source: {source}\n"
-                f"Source ID: {source_id}\n"
-                f"Captured: {now}\n\n"
-                f"---\n\n{truncated}\n\n---\n\n"
-                f"Extract context as JSON:"
-            )
-        }]
+        messages=[
+            {"role": "system", "content": EXTRACTION_SYSTEM_PROMPT},
+            {
+                "role": "user",
+                "content": (
+                    f"Source: {source}\n"
+                    f"Source ID: {source_id}\n"
+                    f"Captured: {now}\n\n"
+                    f"---\n\n{truncated}\n\n---\n\n"
+                    f"Extract context as JSON:"
+                ),
+            },
+        ],
+        # Drop provider-unsupported params so one config works across providers.
+        drop_params=True,
     )
 
-    raw_output = response.content[0].text.strip()
+    raw_output = (response.choices[0].message.content or "").strip()
     extracted = _parse_json_response(raw_output)
 
     # Ensure required fields

requirements.txt

@@ -1,4 +1,5 @@
 anthropic>=0.40.0
+litellm>=1.89.0,<2.0.0
 mcp>=1.0.0
 redis[hiredis]>=5.0.0
 voyageai>=0.2.0

ui.py

@@ -16,10 +16,14 @@
 from instrumentation import setup_tracing
 setup_tracing(project_name="cortex")
 
-from anthropic import Anthropic
+import litellm
 from redis_store import search_context, get_recent_context
 
-client = Anthropic(api_key=os.environ.get("ANTHROPIC_API_KEY"))
+# Model routed through LiteLLM: keep Claude as the default, but any provider
+# LiteLLM supports works by setting CORTEX_MODEL (e.g. "openai/gpt-4o",
+# "gemini/gemini-2.5-pro", "bedrock/..."). Credentials come from that
+# provider's own env var (ANTHROPIC_API_KEY by default).
+DEFAULT_MODEL = os.environ.get("CORTEX_MODEL", "claude-opus-4-5")
 
 # ── Page config ───────────────────────────────────────────────────────────────
 
@@ -119,24 +123,29 @@ def ask_cortex(question: str, context_chunks: list[dict]) -> str:
         for c in context_chunks
     ])
 
-    response = client.messages.create(
-        model="claude-opus-4-5",
-        max_tokens=1000,
-        system="""You are Cortex — a personal AI that knows everything about the user based on their captured context.
+    system_prompt = """You are Cortex — a personal AI that knows everything about the user based on their captured context.
 
 You have access to the user's second brain: notes, decisions, insights, and memories captured from their AI chats, Slack, iMessage, and other apps.
 
 Answer questions directly and personally, as if you are their most knowledgeable assistant.
 - Reference specific details from the context (dates, sources, exact decisions)
 - Be concise but complete
 - If the context is partial, say so and answer with what you have
-- Never say "based on the provided context" — just answer naturally""",
-        messages=[{
-            "role": "user",
-            "content": f"Context from my second brain:\n\n{context_text}\n\n---\n\nQuestion: {question}"
-        }]
+- Never say "based on the provided context" — just answer naturally"""
+    response = litellm.completion(
+        model=DEFAULT_MODEL,
+        max_tokens=1000,
+        messages=[
+            {"role": "system", "content": system_prompt},
+            {
+                "role": "user",
+                "content": f"Context from my second brain:\n\n{context_text}\n\n---\n\nQuestion: {question}",
+            },
+        ],
+        # Drop provider-unsupported params so one config works across providers.
+        drop_params=True,
     )
-    return response.content[0].text
+    return response.choices[0].message.content or ""
 
 
 # ── Sidebar ───────────────────────────────────────────────────────────────────