docker-compose.stack.yml

services:
  model-router:
    build:
      context: .
    restart: unless-stopped
    ports:
      - "${MODEL_ROUTER_PORT:-4000}:4000"
    environment:
      MODEL_ROUTER_ENV: ${MODEL_ROUTER_ENV:-production}
      MODEL_ROUTER_DATABASE_URL: sqlite:////data/model_router.db
      MODEL_ROUTER_JWT_SECRET_KEY: ${MODEL_ROUTER_JWT_SECRET_KEY:-replace-with-a-long-random-secret}
      MODEL_ROUTER_ACCESS_TOKEN_TTL_MINUTES: ${MODEL_ROUTER_ACCESS_TOKEN_TTL_MINUTES:-15}
      MODEL_ROUTER_REFRESH_TOKEN_TTL_DAYS: ${MODEL_ROUTER_REFRESH_TOKEN_TTL_DAYS:-30}
      MODEL_ROUTER_REFRESH_COOKIE_SECURE: ${MODEL_ROUTER_REFRESH_COOKIE_SECURE:-false}
      MODEL_ROUTER_CORS_ORIGINS: ${MODEL_ROUTER_CORS_ORIGINS:-["http://localhost:3000","http://127.0.0.1:3000"]}
      MODEL_ROUTER_BOOTSTRAP_ADMIN_EMAIL: ${MODEL_ROUTER_BOOTSTRAP_ADMIN_EMAIL:-admin@example.com}
      MODEL_ROUTER_BOOTSTRAP_ADMIN_PASSWORD: ${MODEL_ROUTER_BOOTSTRAP_ADMIN_PASSWORD:-change-me-now}
      MODEL_ROUTER_BOOTSTRAP_TENANT_NAME: ${MODEL_ROUTER_BOOTSTRAP_TENANT_NAME:-Default Tenant}
      MODEL_ROUTER_BOOTSTRAP_TENANT_SLUG: ${MODEL_ROUTER_BOOTSTRAP_TENANT_SLUG:-default}
      MODEL_ROUTER_BOOTSTRAP_ROUTES_JSON: ${MODEL_ROUTER_BOOTSTRAP_ROUTES_JSON:-[]}
      MODEL_ROUTER_BOOTSTRAP_API_KEYS_JSON: ${MODEL_ROUTER_BOOTSTRAP_API_KEYS_JSON:-[]}
      # Managed llama-server via Docker wrapper
      MODEL_ROUTER_MANAGED_LLAMA_COMMAND: /usr/local/bin/llama-server
      MODEL_ROUTER_MANAGED_LLAMA_PORT: 8090
      MODEL_ROUTER_MANAGED_LLAMA_IDLE_TIMEOUT_SECONDS: 120
      MODEL_ROUTER_MANAGED_LLAMA_STARTUP_TIMEOUT_SECONDS: 300
      MODEL_ROUTER_MANAGED_LLAMA_SHUTDOWN_TIMEOUT_SECONDS: 35
      MODEL_ROUTER_MANAGED_LLAMA_MODELS_JSON: '${MODEL_ROUTER_MANAGED_LLAMA_MODELS_JSON:-{"Hui-gemma-26B-A4B-abb-IQ4_XS.gguf":{"model_path":"/models/gemma-4-26B-A4B-it-abliterated.IQ4_XS.gguf","cli_args":["-c","65536","-b","2048","-ub","512","-t","8","--cache-type-k","turbo4","--cache-type-v","turbo3"],"idle_timeout_seconds":120},"gemma-4-31B-it-qat-UD-Q4_K_XL.gguf":{"model_path":"/models/gemma-4-31B-it-qat-UD-Q4_K_XL.gguf","cli_args":["-c","192000","-b","2048","-ub","512","-t","8","--cache-type-k","turbo4","--cache-type-v","turbo3"],"idle_timeout_seconds":300}}}'
      # Wrapper + llama-cpp need access to host models
      MODELS_HOST_DIR: ${MODELS_HOST_DIR:-/home/kleber/models}
    volumes:
      - ${MODEL_ROUTER_DATA_DIR:-./runtime/model-router}:/data
      - /home/kleber/model-router/llama-server-docker-wrapper.sh:/usr/local/bin/llama-server:ro
      - /usr/bin/docker:/usr/bin/docker:ro
      - /home/kleber/models:/models:ro
      - /var/run/docker.sock:/var/run/docker.sock:rw
    healthcheck:
      test:
        - CMD
        - python
        - -c
        - import urllib.request; urllib.request.urlopen("http://127.0.0.1:4000/readyz").read()
      interval: 10s
      timeout: 5s
      retries: 12
    networks:
      - ai-stack

  hermes:
    image: ${HERMES_IMAGE:-nousresearch/hermes-agent:latest}
    restart: unless-stopped
    command: gateway run
    environment:
      HERMES_UID: ${HERMES_UID:-1000}
      HERMES_GID: ${HERMES_GID:-1000}
      API_SERVER_ENABLED: "true"
      API_SERVER_HOST: 0.0.0.0
      API_SERVER_PORT: 8642
      API_SERVER_KEY: hermes-sk-local-dev
      API_SERVER_MODEL_NAME: ${HERMES_API_MODEL_NAME:-hermes-agent}
      MODEL_ROUTER_API_KEY: ${OPEN_WEBUI_ROUTER_API_KEY}
    volumes:
      - ${HERMES_HOME:-./runtime/hermes}:/opt/data
    networks:
      - ai-stack

  open-webui:
    build:
      context: ./open-webui-overlay
      args:
        OPEN_WEBUI_BASE_IMAGE: ${OPEN_WEBUI_IMAGE:-ghcr.io/open-webui/open-webui:main}
    image: ${OPEN_WEBUI_PATCHED_IMAGE:-open-webui-router:local}
    restart: unless-stopped
    depends_on:
      model-router:
        condition: service_healthy
      db:
        condition: service_healthy
    ports:
      - "${OPEN_WEBUI_PORT:-3000}:8080"
    environment:
      WEBUI_SECRET_KEY: ${WEBUI_SECRET_KEY:-opswbsecret2026changeme}
      ENABLE_WEB_SEARCH: ${ENABLE_WEB_SEARCH:-true}
      WEB_SEARCH_ENGINE: ${WEB_SEARCH_ENGINE:-searxng}
      SEARXNG_QUERY_URL: ${SEARXNG_QUERY_URL:-http://searxng:8080}
      OPENAI_API_BASE_URL: http://model-router:4000/v1
      OPENAI_API_KEY: ${OPEN_WEBUI_ROUTER_API_KEY}
      OLLAMA_BASE_URL: http://127.0.0.1:9
      BYPASS_WEB_SEARCH_WEB_LOADER: ${BYPASS_WEB_SEARCH_WEB_LOADER:-true}
      BYPASS_RETRIEVAL_ACCESS_CONTROL: ${BYPASS_RETRIEVAL_ACCESS_CONTROL:-true}
      RAG_TOP_K: ${RAG_TOP_K:-10}
      RAG_RELEVANCE_THRESHOLD: ${RAG_RELEVANCE_THRESHOLD:-0.0}
      WHISPER_MODEL: ${WHISPER_MODEL:-large-v3-turbo}
      WHISPER_MULTILINGUAL: ${WHISPER_MULTILINGUAL:-true}
      AUDIO_TTS_ENGINE: ${AUDIO_TTS_ENGINE:-openai}
      AUDIO_TTS_MODEL: ${AUDIO_TTS_MODEL:-kokoro}
      AUDIO_TTS_VOICE: ${AUDIO_TTS_VOICE:-af_heart}
      AUDIO_TTS_OPENAI_API_BASE_URL: ${AUDIO_TTS_OPENAI_API_BASE_URL:-http://kokoro-tts:8881/v1}
      AUDIO_TTS_OPENAI_API_KEY: ${AUDIO_TTS_OPENAI_API_KEY:-not-needed}
      AUDIO_TTS_SPLIT_ON: ${AUDIO_TTS_SPLIT_ON:-none}
      TERMINAL_SERVER_CONNECTIONS: ${TERMINAL_SERVER_CONNECTIONS:-[{"id":"local","name":"Local Shell","url":"http://ttyd:7681","enabled":true}]}
      SYSTEM_PROMPT: ${SYSTEM_PROMPT:-}
      OPENWEBUI_INLINE_FULL_CONTEXT_MAX_CHARS: ${OPENWEBUI_INLINE_FULL_CONTEXT_MAX_CHARS:-16000}
      DATABASE_URL: ${DATABASE_URL:-postgresql+psycopg2://openwebui:openwebui@db:5432/openwebui}
      VECTOR_DB: ${VECTOR_DB:-pgvector}
      PGVECTOR_DB_URL: ${PGVECTOR_DB_URL:-postgresql+psycopg2://openwebui:openwebui@db:5432/openwebui}
    volumes:
      - ${OPEN_WEBUI_DATA_DIR:-./runtime/open-webui}:/app/backend/data
    networks:
      - ai-stack
      - webscraper-agent_default

  kokoro-tts:
    build:
      context: ./kokoro-tts
    restart: unless-stopped
    environment:
      HF_HOME: /data/hf
    volumes:
      - kokoro_data:/data
    networks:
      - ai-stack

  ttyd:
    image: tsl0922/ttyd:latest
    restart: unless-stopped
    command: ttyd --port 7681 --writable bash
    networks:
      - ai-stack

  db:
    image: pgvector/pgvector:pg17
    restart: unless-stopped
    environment:
      POSTGRES_USER: ${POSTGRES_USER:-openwebui}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-openwebui}
      POSTGRES_DB: ${POSTGRES_DB:-openwebui}
    volumes:
      - pgdata:/var/lib/postgresql/data
    networks:
      - ai-stack
    healthcheck:
      test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER:-openwebui}"]
      interval: 10s
      retries: 5

  searxng:
    image: searxng/searxng:latest
    restart: unless-stopped
    ports:
      - "${SEARXNG_PORT:-8088}:8080"
    environment:
      SEARXNG_BASE_URL: ${SEARXNG_BASE_URL:-}
    volumes:
      - ./searxng-conf:/etc/searxng:ro
    networks:
      - ai-stack
    cap_drop:
      - ALL
    cap_add:
      - CHOWN
      - SETGID
      - SETUID

  llama-cpp:
    image: ${LLAMA_CPP_IMAGE:-vito974/llama-cpp-turboquant:server-cuda12}
    entrypoint: ["/bin/bash", "-lc"]
    restart: unless-stopped
    profiles:
      - llama
    gpus: all
    ports:
      - "${LLAMA_PORT:-8081}:8080"
    environment:
      LLAMA_MODEL: ${LLAMA_MODEL:-gemma-4-26B-A4B-it-abliterated}
      LLAMA_MODEL_FILE: ${LLAMA_MODEL_FILE:-/models/gemma-4-26B-A4B-it-abliterated.IQ4_XS.gguf}
      LLAMA_CTX_SIZE: ${LLAMA_CTX_SIZE:-65536}
      LLAMA_CACHE_TYPE_K: ${LLAMA_CACHE_TYPE_K:-turbo4}
      LLAMA_CACHE_TYPE_V: ${LLAMA_CACHE_TYPE_V:-turbo3}
      LLAMA_BATCH_SIZE: ${LLAMA_BATCH_SIZE:-2048}
      LLAMA_UBATCH_SIZE: ${LLAMA_UBATCH_SIZE:-512}
      LLAMA_THREADS: ${LLAMA_THREADS:-8}
      LLAMA_N_CPU_MOE: ${LLAMA_N_CPU_MOE:-0}
      HF_HOME: /data/hf
      XDG_CACHE_HOME: /data/.cache
    volumes:
      - ${LLAMA_CACHE_DIR:-./runtime/llama-cpp}:/data
      - ${MODELS_HOST_DIR:-/home/kleber/models}:/models:ro
    command:
      - >-
        set -euo pipefail;
        args=(
          /app/llama-server
          -m "$${LLAMA_MODEL_FILE}"
          -ngl all
          -ncmoe "$${LLAMA_N_CPU_MOE}"
          --no-mmap
          --mlock
          --cache-type-k "$${LLAMA_CACHE_TYPE_K}"
          --cache-type-v "$${LLAMA_CACHE_TYPE_V}"
          -b "$${LLAMA_BATCH_SIZE}"
          -ub "$${LLAMA_UBATCH_SIZE}"
          -t "$${LLAMA_THREADS}"
          -fa on
          -c "$${LLAMA_CTX_SIZE}"
          --host 0.0.0.0
          --port 8080
        );
        exec "$${args[@]}"
    networks:
      - ai-stack

  startup-checks:
    build:
      context: .
    entrypoint: ["python3", "/app/scripts/startup_checks.py"]
    environment:
      MODEL_ROUTER_API_KEY: ${OPEN_WEBUI_ROUTER_API_KEY}
      PG_DSN: ${PG_DSN:-postgresql://openwebui:openwebui@db:5432/openwebui}
    depends_on:
      model-router:
        condition: service_healthy
      db:
        condition: service_healthy
    volumes:
      - ./scripts:/app/scripts:ro
      - ${MODEL_ROUTER_DATA_DIR:-./runtime/model-router}:/data
    networks:
      - ai-stack
    restart: "no"

  litestream:
    image: litestream/litestream:latest
    user: "0:0"
    restart: unless-stopped
    depends_on:
      open-webui:
        condition: service_started
    command: replicate -config /etc/litestream.yml
    environment:
      LITESTREAM_S3_ACCESS_KEY_ID: ${LITESTREAM_S3_ACCESS_KEY_ID:-minio}
      LITESTREAM_S3_SECRET_ACCESS_KEY: ${LITESTREAM_S3_SECRET_ACCESS_KEY:-miniosecret}
    volumes:
      - ./litestream.yml:/etc/litestream.yml:ro
      - ${OPEN_WEBUI_DATA_DIR:-./runtime/open-webui}:/data
    networks:
      - ai-stack
      - langfuse_default

volumes:
  kokoro_data:
  pgdata:

networks:
  ai-stack:
    driver: bridge
  webscraper-agent_default:
    external: true
  langfuse_default:
    external: true