diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000000..a111dbfdbb7 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://raw.githubusercontent.com/devcontainers/spec/main/schemas/devContainer.schema.json", + "name": "LocalAI", + "workspaceFolder": "/workspace", + "dockerComposeFile": [ "./docker-compose-devcontainer.yml" ], + "service": "api", + "shutdownAction": "stopCompose", + "customizations": { + "vscode": { + "extensions": [ + "golang.go", + "ms-vscode.makefile-tools", + "ms-azuretools.vscode-docker", + "ms-python.python", + "ms-python.debugpy", + "wayou.vscode-todo-highlight", + "waderyan.gitblame" + ] + } + }, + "forwardPorts": [8080, 3000], + "postStartCommand": "make prepare && cp /build/backend-assets /workdir/backend-assets" +} \ No newline at end of file diff --git a/.devcontainer/docker-compose-devcontainer.yml b/.devcontainer/docker-compose-devcontainer.yml new file mode 100644 index 00000000000..e36492e9f0b --- /dev/null +++ b/.devcontainer/docker-compose-devcontainer.yml @@ -0,0 +1,45 @@ +services: + api: + build: + context: .. + dockerfile: Dockerfile + target: devcontainer + args: + - FFMPEG=true + - IMAGE_TYPE=extras + - GO_TAGS=stablediffusion p2p tts + env_file: + - ../.env + ports: + - 8080:8080 + volumes: + - ..:/workspace:cached + command: /bin/sh -c "while sleep 1000; do :; done" + cap_add: + - SYS_PTRACE + security_opt: + - seccomp:unconfined + prometheus: + image: prom/prometheus + container_name: prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + ports: + - 9090:9090 + restart: unless-stopped + volumes: + - ./prometheus:/etc/prometheus + - prom_data:/prometheus + grafana: + image: grafana/grafana + container_name: grafana + ports: + - 3000:3000 + restart: unless-stopped + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=grafana + volumes: + - ./grafana:/etc/grafana/provisioning/datasources +volumes: + prom_data: \ No newline at end of file diff --git a/.devcontainer/grafana/datasource.yml b/.devcontainer/grafana/datasource.yml new file mode 100644 index 00000000000..1ed2fa3c2a2 --- /dev/null +++ b/.devcontainer/grafana/datasource.yml @@ -0,0 +1,10 @@ + +apiVersion: 1 + +datasources: +- name: Prometheus + type: prometheus + url: http://prometheus:9090 + isDefault: true + access: proxy + editable: true diff --git a/.devcontainer/prometheus/prometheus.yml b/.devcontainer/prometheus/prometheus.yml new file mode 100644 index 00000000000..18c44da7144 --- /dev/null +++ b/.devcontainer/prometheus/prometheus.yml @@ -0,0 +1,21 @@ +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s +alerting: + alertmanagers: + - static_configs: + - targets: [] + scheme: http + timeout: 10s + api_version: v1 +scrape_configs: +- job_name: prometheus + honor_timestamps: true + scrape_interval: 15s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + static_configs: + - targets: + - localhost:9090 \ No newline at end of file diff --git a/.dockerignore b/.dockerignore index 3954769f5c3..e91f0008f1c 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,7 @@ .idea .github .vscode +.devcontainer models examples/chatbot-ui/models examples/rwkv/models diff --git a/.env b/.env index 95a515bc850..9e5dbd79ee4 100644 --- a/.env +++ b/.env @@ -79,6 +79,9 @@ ### Enable to run parallel requests # LOCALAI_PARALLEL_REQUESTS=true +# Enable to allow p2p mode +# LOCALAI_P2P=true + ### Watchdog settings ### # Enables watchdog to kill backends that are inactive for too much time diff --git a/.vscode/launch.json b/.vscode/launch.json index 2727da924b0..504934210b3 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -3,12 +3,12 @@ "configurations": [ { "name": "Python: Current File", - "type": "python", + "type": "debugpy", "request": "launch", "program": "${file}", "console": "integratedTerminal", "justMyCode": false, - "cwd": "${workspaceFolder}/examples/langchain-chroma", + "cwd": "${fileDirname}", "env": { "OPENAI_API_BASE": "http://localhost:8080/v1", "OPENAI_API_KEY": "abc" @@ -19,15 +19,16 @@ "type": "go", "request": "launch", "mode": "debug", - "program": "${workspaceFolder}/main.go", - "args": [ - "api" - ], + "program": "${workspaceRoot}", + "args": [], "env": { - "C_INCLUDE_PATH": "${workspaceFolder}/go-llama:${workspaceFolder}/go-stable-diffusion/:${workspaceFolder}/gpt4all/gpt4all-bindings/golang/:${workspaceFolder}/go-gpt2:${workspaceFolder}/go-rwkv:${workspaceFolder}/whisper.cpp:${workspaceFolder}/go-bert:${workspaceFolder}/bloomz", - "LIBRARY_PATH": "${workspaceFolder}/go-llama:${workspaceFolder}/go-stable-diffusion/:${workspaceFolder}/gpt4all/gpt4all-bindings/golang/:${workspaceFolder}/go-gpt2:${workspaceFolder}/go-rwkv:${workspaceFolder}/whisper.cpp:${workspaceFolder}/go-bert:${workspaceFolder}/bloomz", - "DEBUG": "true" - } + "LOCALAI_LOG_LEVEL": "debug", + "LOCALAI_P2P": "true", + "LOCALAI_FEDERATED": "true" + }, + "buildFlags": ["-tags", "stablediffusion p2p tts", "-v"], + "envFile": "${workspaceFolder}/.env", + "cwd": "${workspaceRoot}" } ] } \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index a0feadd9c7e..0dfaaa19224 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ FROM ${BASE_IMAGE} AS requirements-core USER root -ARG GO_VERSION=1.22.5 +ARG GO_VERSION=1.22.6 ARG TARGETARCH ARG TARGETVARIANT @@ -30,7 +30,7 @@ RUN apt-get update && \ # Install Go RUN curl -L -s https://go.dev/dl/go${GO_VERSION}.linux-${TARGETARCH}.tar.gz | tar -C /usr/local -xz -ENV PATH $PATH:/root/go/bin:/usr/local/go/bin +ENV PATH=$PATH:/root/go/bin:/usr/local/go/bin # Install grpc compilers RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 && \ @@ -39,15 +39,18 @@ RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2 && \ COPY --chmod=644 custom-ca-certs/* /usr/local/share/ca-certificates/ RUN update-ca-certificates +RUN test -n "$TARGETARCH" \ + || (echo 'warn: missing $TARGETARCH, either set this `ARG` manually, or run using `docker buildkit`') + # Use the variables in subsequent instructions RUN echo "Target Architecture: $TARGETARCH" RUN echo "Target Variant: $TARGETVARIANT" # Cuda -ENV PATH /usr/local/cuda/bin:${PATH} +ENV PATH=/usr/local/cuda/bin:${PATH} # HipBLAS requirements -ENV PATH /opt/rocm/bin:${PATH} +ENV PATH=/opt/rocm/bin:${PATH} # OpenBLAS requirements and stable diffusion RUN apt-get update && \ @@ -62,9 +65,6 @@ RUN ln -s /usr/include/opencv4/opencv2 /usr/include/opencv2 WORKDIR /build -RUN test -n "$TARGETARCH" \ - || (echo 'warn: missing $TARGETARCH, either set this `ARG` manually, or run using `docker buildkit`') - ################################### ################################### @@ -217,13 +217,14 @@ RUN git clone --recurse-submodules --jobs 4 -b ${GRPC_VERSION} --depth 1 --shall ################################### ################################### -# The builder target compiles LocalAI. This target is not the target that will be uploaded to the registry. -# Adjustments to the build process should likely be made here. -FROM requirements-drivers AS builder +# The builder-base target has the arguments, variables, and copies shared between full builder images and the uncompiled devcontainer + +FROM requirements-drivers AS builder-base ARG GO_TAGS="stablediffusion tts p2p" ARG GRPC_BACKENDS ARG MAKEFLAGS +ARG LD_FLAGS="-s -w" ENV GRPC_BACKENDS=${GRPC_BACKENDS} ENV GO_TAGS=${GO_TAGS} @@ -231,14 +232,12 @@ ENV MAKEFLAGS=${MAKEFLAGS} ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility ENV NVIDIA_REQUIRE_CUDA="cuda>=${CUDA_MAJOR_VERSION}.0" ENV NVIDIA_VISIBLE_DEVICES=all +ENV LD_FLAGS=${LD_FLAGS} -WORKDIR /build +RUN echo "GO_TAGS: $GO_TAGS" && echo "TARGETARCH: $TARGETARCH" -COPY . . -COPY .git . -RUN echo "GO_TAGS: $GO_TAGS" +WORKDIR /build -RUN make prepare # We need protoc installed, and the version in 22.04 is too old. We will create one as part installing the GRPC build below # but that will also being in a newer version of absl which stablediffusion cannot compile with. This version of protoc is only @@ -256,6 +255,20 @@ RUN <