Skip to content

Commit e15d29a

Browse files
mudlermudler
authored
chore(stablediffusion-ncn): drop in favor of ggml implementation (#4652)
* chore(stablediffusion-ncn): drop in favor of ggml implementation Signed-off-by: Ettore Di Giacinto <[email protected]> * chore(ci): drop stablediffusion build Signed-off-by: Ettore Di Giacinto <[email protected]> * chore(tests): add Signed-off-by: Ettore Di Giacinto <[email protected]> * chore(tests): try to fixup current tests Signed-off-by: Ettore Di Giacinto <[email protected]> * Try to fix tests Signed-off-by: Ettore Di Giacinto <[email protected]> * Tests improvements Signed-off-by: Ettore Di Giacinto <[email protected]> * chore(tests): use quality to specify step Signed-off-by: Ettore Di Giacinto <[email protected]> * chore(tests): switch to sd-1.5 also increase prep time for downloading models Signed-off-by: Ettore Di Giacinto <[email protected]> --------- Signed-off-by: Ettore Di Giacinto <[email protected]>
1 parent 10675ac commit e15d29a

File tree

22 files changed

+123
-302
lines changed

22 files changed

+123
-302
lines changed

‎.devcontainer/docker-compose-devcontainer.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ services:
77
args:
88
- FFMPEG=true
99
- IMAGE_TYPE=extras
10-
- GO_TAGS=stablediffusion p2p tts
10+
- GO_TAGS=p2p tts
1111
env_file:
1212
- ../.env
1313
ports:

‎.env

+3-3
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,12 @@
3838
## Uncomment and set to true to enable rebuilding from source
3939
# REBUILD=true
4040

41-
## Enable go tags, available: stablediffusion, tts
42-
## stablediffusion: image generation with stablediffusion
41+
## Enable go tags, available: p2p, tts
42+
## p2p: enable distributed inferencing
4343
## tts: enables text-to-speech with go-piper
4444
## (requires REBUILD=true)
4545
#
46-
# GO_TAGS=stablediffusion
46+
# GO_TAGS=p2p
4747

4848
## Path where to store generated images
4949
# LOCALAI_IMAGE_PATH=/tmp/generated/images

‎.github/workflows/release.yaml

+1-34
Original file line numberDiff line numberDiff line change
@@ -237,40 +237,7 @@ jobs:
237237
detached: true
238238
connect-timeout-seconds: 180
239239
limit-access-to-actor: true
240-
build-stablediffusion:
241-
runs-on: ubuntu-latest
242-
steps:
243-
- name: Clone
244-
uses: actions/checkout@v4
245-
with:
246-
submodules: true
247-
- uses: actions/setup-go@v5
248-
with:
249-
go-version: '1.21.x'
250-
cache: false
251-
- name: Dependencies
252-
run: |
253-
sudo apt-get update
254-
sudo apt-get install -y --no-install-recommends libopencv-dev protobuf-compiler ccache upx-ucl
255-
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af
256-
go install google.golang.org/protobuf/cmd/[email protected]
257-
- name: Build stablediffusion
258-
run: |
259-
export PATH=$PATH:$GOPATH/bin
260-
make backend-assets/grpc/stablediffusion
261-
mkdir -p release && cp backend-assets/grpc/stablediffusion release
262-
env:
263-
GO_TAGS: stablediffusion
264-
- uses: actions/upload-artifact@v4
265-
with:
266-
name: stablediffusion
267-
path: release/
268-
- name: Release
269-
uses: softprops/action-gh-release@v2
270-
if: startsWith(github.ref, 'refs/tags/')
271-
with:
272-
files: |
273-
release/*
240+
274241

275242
build-macOS-x86_64:
276243
runs-on: macos-13

‎.github/workflows/test.yml

+2-4
Original file line numberDiff line numberDiff line change
@@ -105,9 +105,7 @@ jobs:
105105
# Pre-build piper before we start tests in order to have shared libraries in place
106106
make sources/go-piper && \
107107
GO_TAGS="tts" make -C sources/go-piper piper.o && \
108-
sudo cp -rfv sources/go-piper/piper-phonemize/pi/lib/. /usr/lib/ && \
109-
# Pre-build stable diffusion before we install a newer version of abseil (not compatible with stablediffusion-ncn)
110-
PATH="$PATH:/root/go/bin" GO_TAGS="stablediffusion tts" GRPC_BACKENDS=backend-assets/grpc/stablediffusion make build
108+
sudo cp -rfv sources/go-piper/piper-phonemize/pi/lib/. /usr/lib/
111109
env:
112110
CUDA_VERSION: 12-4
113111
- name: Cache grpc
@@ -129,7 +127,7 @@ jobs:
129127
cd grpc && cd cmake/build && sudo make --jobs 5 install
130128
- name: Test
131129
run: |
132-
PATH="$PATH:/root/go/bin" GO_TAGS="stablediffusion tts" make --jobs 5 --output-sync=target test
130+
PATH="$PATH:/root/go/bin" GO_TAGS="tts" make --jobs 5 --output-sync=target test
133131
- name: Setup tmate session if tests fail
134132
if: ${{ failure() }}
135133
uses: mxschmitt/[email protected]

‎.vscode/launch.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@
2626
"LOCALAI_P2P": "true",
2727
"LOCALAI_FEDERATED": "true"
2828
},
29-
"buildFlags": ["-tags", "stablediffusion p2p tts", "-v"],
29+
"buildFlags": ["-tags", "p2p tts", "-v"],
3030
"envFile": "${workspaceFolder}/.env",
3131
"cwd": "${workspaceRoot}"
3232
}

‎Dockerfile

+3-35
Original file line numberDiff line numberDiff line change
@@ -69,14 +69,10 @@ ENV PATH=/opt/rocm/bin:${PATH}
6969
# OpenBLAS requirements and stable diffusion
7070
RUN apt-get update && \
7171
apt-get install -y --no-install-recommends \
72-
libopenblas-dev \
73-
libopencv-dev && \
72+
libopenblas-dev && \
7473
apt-get clean && \
7574
rm -rf /var/lib/apt/lists/*
7675

77-
# Set up OpenCV
78-
RUN ln -s /usr/include/opencv4/opencv2 /usr/include/opencv2
79-
8076
WORKDIR /build
8177

8278
###################################
@@ -251,7 +247,7 @@ RUN git clone --recurse-submodules --jobs 4 -b ${GRPC_VERSION} --depth 1 --shall
251247

252248
FROM requirements-drivers AS builder-base
253249

254-
ARG GO_TAGS="stablediffusion tts p2p"
250+
ARG GO_TAGS="tts p2p"
255251
ARG GRPC_BACKENDS
256252
ARG MAKEFLAGS
257253
ARG LD_FLAGS="-s -w"
@@ -285,35 +281,12 @@ RUN <<EOT bash
285281
fi
286282
EOT
287283

288-
289-
###################################
290-
###################################
291-
292-
# This first portion of builder holds the layers specifically used to build backend-assets/grpc/stablediffusion
293-
# In most cases, builder is the image you should be using - however, this can save build time if one just needs to copy backend-assets/grpc/stablediffusion and nothing else.
294-
FROM builder-base AS builder-sd
295-
296-
# stablediffusion does not tolerate a newer version of abseil, copy only over enough elements to build it
297-
COPY Makefile .
298-
COPY go.mod .
299-
COPY go.sum .
300-
COPY backend/backend.proto ./backend/backend.proto
301-
COPY backend/go/image/stablediffusion ./backend/go/image/stablediffusion
302-
COPY pkg/grpc ./pkg/grpc
303-
COPY pkg/stablediffusion ./pkg/stablediffusion
304-
RUN git init
305-
RUN make sources/go-stable-diffusion
306-
RUN touch prepare-sources
307-
308-
# Actually build the backend
309-
RUN GRPC_BACKENDS=backend-assets/grpc/stablediffusion make backend-assets/grpc/stablediffusion
310-
311284
###################################
312285
###################################
313286

314287
# The builder target compiles LocalAI. This target is not the target that will be uploaded to the registry.
315288
# Adjustments to the build process should likely be made here.
316-
FROM builder-sd AS builder
289+
FROM builder-base AS builder
317290

318291
# Install the pre-built GRPC
319292
COPY --from=grpc /opt/grpc /usr/local
@@ -353,8 +326,6 @@ ARG FFMPEG
353326

354327
COPY --from=grpc /opt/grpc /usr/local
355328

356-
COPY --from=builder-sd /build/backend-assets/grpc/stablediffusion /build/backend-assets/grpc/stablediffusion
357-
358329
COPY .devcontainer-scripts /.devcontainer-scripts
359330

360331
# Add FFmpeg
@@ -427,9 +398,6 @@ COPY --from=builder /build/local-ai ./
427398
# Copy shared libraries for piper
428399
COPY --from=builder /build/sources/go-piper/piper-phonemize/pi/lib/* /usr/lib/
429400

430-
# do not let stablediffusion rebuild (requires an older version of absl)
431-
COPY --from=builder-sd /build/backend-assets/grpc/stablediffusion ./backend-assets/grpc/stablediffusion
432-
433401
# Change the shell to bash so we can use [[ tests below
434402
SHELL ["/bin/bash", "-c"]
435403
# We try to strike a balance between individual layer size (as that affects total push time) and total image size

‎Makefile

+2-34
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,6 @@ WHISPER_CPP_VERSION?=6266a9f9e56a5b925e9892acf650f3eb1245814d
1818
PIPER_REPO?=https://github.com/mudler/go-piper
1919
PIPER_VERSION?=e10ca041a885d4a8f3871d52924b47792d5e5aa0
2020

21-
# stablediffusion version
22-
STABLEDIFFUSION_REPO?=https://github.com/mudler/go-stable-diffusion
23-
STABLEDIFFUSION_VERSION?=4a3cd6aeae6f66ee57eae9a0075f8c58c3a6a38f
24-
2521
# bark.cpp
2622
BARKCPP_REPO?=https://github.com/PABannier/bark.cpp.git
2723
BARKCPP_VERSION?=v1.0.0
@@ -179,11 +175,6 @@ ifeq ($(STATIC),true)
179175
LD_FLAGS+=-linkmode external -extldflags -static
180176
endif
181177

182-
ifeq ($(findstring stablediffusion,$(GO_TAGS)),stablediffusion)
183-
# OPTIONAL_TARGETS+=go-stable-diffusion/libstablediffusion.a
184-
OPTIONAL_GRPC+=backend-assets/grpc/stablediffusion
185-
endif
186-
187178
ifeq ($(findstring tts,$(GO_TAGS)),tts)
188179
# OPTIONAL_TARGETS+=go-piper/libpiper_binding.a
189180
# OPTIONAL_TARGETS+=backend-assets/espeak-ng-data
@@ -273,19 +264,6 @@ sources/go-piper:
273264
sources/go-piper/libpiper_binding.a: sources/go-piper
274265
$(MAKE) -C sources/go-piper libpiper_binding.a example/main piper.o
275266

276-
## stable diffusion (onnx)
277-
sources/go-stable-diffusion:
278-
mkdir -p sources/go-stable-diffusion
279-
cd sources/go-stable-diffusion && \
280-
git init && \
281-
git remote add origin $(STABLEDIFFUSION_REPO) && \
282-
git fetch origin && \
283-
git checkout $(STABLEDIFFUSION_VERSION) && \
284-
git submodule update --init --recursive --depth 1 --single-branch
285-
286-
sources/go-stable-diffusion/libstablediffusion.a: sources/go-stable-diffusion
287-
CPATH="$(CPATH):/usr/include/opencv4" $(MAKE) -C sources/go-stable-diffusion libstablediffusion.a
288-
289267
## stablediffusion (ggml)
290268
sources/stablediffusion-ggml.cpp:
291269
git clone --recursive $(STABLEDIFFUSION_GGML_REPO) sources/stablediffusion-ggml.cpp && \
@@ -331,20 +309,18 @@ sources/whisper.cpp:
331309
sources/whisper.cpp/libwhisper.a: sources/whisper.cpp
332310
cd sources/whisper.cpp && $(MAKE) libwhisper.a libggml.a
333311

334-
get-sources: sources/go-llama.cpp sources/go-piper sources/stablediffusion-ggml.cpp sources/bark.cpp sources/whisper.cpp sources/go-stable-diffusion backend/cpp/llama/llama.cpp
312+
get-sources: sources/go-llama.cpp sources/go-piper sources/stablediffusion-ggml.cpp sources/bark.cpp sources/whisper.cpp backend/cpp/llama/llama.cpp
335313

336314
replace:
337315
$(GOCMD) mod edit -replace github.com/ggerganov/whisper.cpp=$(CURDIR)/sources/whisper.cpp
338316
$(GOCMD) mod edit -replace github.com/ggerganov/whisper.cpp/bindings/go=$(CURDIR)/sources/whisper.cpp/bindings/go
339317
$(GOCMD) mod edit -replace github.com/mudler/go-piper=$(CURDIR)/sources/go-piper
340-
$(GOCMD) mod edit -replace github.com/mudler/go-stable-diffusion=$(CURDIR)/sources/go-stable-diffusion
341318
$(GOCMD) mod edit -replace github.com/go-skynet/go-llama.cpp=$(CURDIR)/sources/go-llama.cpp
342319

343320
dropreplace:
344321
$(GOCMD) mod edit -dropreplace github.com/ggerganov/whisper.cpp
345322
$(GOCMD) mod edit -dropreplace github.com/ggerganov/whisper.cpp/bindings/go
346323
$(GOCMD) mod edit -dropreplace github.com/mudler/go-piper
347-
$(GOCMD) mod edit -dropreplace github.com/mudler/go-stable-diffusion
348324
$(GOCMD) mod edit -dropreplace github.com/go-skynet/go-llama.cpp
349325

350326
prepare-sources: get-sources replace
@@ -355,7 +331,6 @@ rebuild: ## Rebuilds the project
355331
$(GOCMD) clean -cache
356332
$(MAKE) -C sources/go-llama.cpp clean
357333
$(MAKE) -C sources/whisper.cpp clean
358-
$(MAKE) -C sources/go-stable-diffusion clean
359334
$(MAKE) -C sources/go-piper clean
360335
$(MAKE) build
361336

@@ -470,7 +445,7 @@ prepare-test: grpcs
470445

471446
test: prepare test-models/testmodel.ggml grpcs
472447
@echo 'Running tests'
473-
export GO_TAGS="tts stablediffusion debug"
448+
export GO_TAGS="tts debug"
474449
$(MAKE) prepare-test
475450
HUGGINGFACE_GRPC=$(abspath ./)/backend/python/transformers/run.sh TEST_DIR=$(abspath ./)/test-dir/ FIXTURES=$(abspath ./)/tests/fixtures CONFIG_FILE=$(abspath ./)/test-models/config.yaml MODELS_PATH=$(abspath ./)/test-models \
476451
$(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter="!llama && !llama-gguf" --flake-attempts $(TEST_FLAKES) --fail-fast -v -r $(TEST_PATHS)
@@ -816,13 +791,6 @@ ifneq ($(UPX),)
816791
$(UPX) backend-assets/grpc/piper
817792
endif
818793

819-
backend-assets/grpc/stablediffusion: sources/go-stable-diffusion sources/go-stable-diffusion/libstablediffusion.a backend-assets/grpc
820-
CGO_LDFLAGS="$(CGO_LDFLAGS)" CPATH="$(CPATH):$(CURDIR)/sources/go-stable-diffusion/:/usr/include/opencv4" LIBRARY_PATH=$(CURDIR)/sources/go-stable-diffusion/ \
821-
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/stablediffusion ./backend/go/image/stablediffusion
822-
ifneq ($(UPX),)
823-
$(UPX) backend-assets/grpc/stablediffusion
824-
endif
825-
826794
backend-assets/grpc/silero-vad: backend-assets/grpc backend-assets/lib/libonnxruntime.so.1
827795
CGO_LDFLAGS="$(CGO_LDFLAGS)" CPATH="$(CPATH):$(CURDIR)/sources/onnxruntime/include/" LIBRARY_PATH=$(CURDIR)/backend-assets/lib \
828796
$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/silero-vad ./backend/go/vad/silero

‎aio/cpu/image-gen.yaml

+10-49
Original file line numberDiff line numberDiff line change
@@ -1,56 +1,17 @@
11
name: stablediffusion
2-
backend: stablediffusion
3-
parameters:
4-
model: stablediffusion_assets
5-
6-
license: "BSD-3"
7-
urls:
8-
- https://github.com/EdVince/Stable-Diffusion-NCNN
9-
- https://github.com/EdVince/Stable-Diffusion-NCNN/blob/main/LICENSE
2+
backend: stablediffusion-ggml
3+
cfg_scale: 4.5
104

11-
description: |
12-
Stable Diffusion in NCNN with c++, supported txt2img and img2img
5+
options:
6+
- sampler:euler
7+
parameters:
8+
model: stable-diffusion-v1-5-pruned-emaonly-Q4_0.gguf
9+
step: 25
1310

1411
download_files:
15-
- filename: "stablediffusion_assets/AutoencoderKL-256-256-fp16-opt.param"
16-
sha256: "18ca4b66685e21406bcf64c484b3b680b4949900415536d599cc876579c85c82"
17-
uri: "https://raw.githubusercontent.com/EdVince/Stable-Diffusion-NCNN/main/x86/linux/assets/AutoencoderKL-256-256-fp16-opt.param"
18-
- filename: "stablediffusion_assets/AutoencoderKL-512-512-fp16-opt.param"
19-
sha256: "cf45f63aacf3dbbab0f59ed92a6f2c14d9a1801314631cd3abe91e3c85639a20"
20-
uri: "https://raw.githubusercontent.com/EdVince/Stable-Diffusion-NCNN/main/x86/linux/assets/AutoencoderKL-512-512-fp16-opt.param"
21-
- filename: "stablediffusion_assets/AutoencoderKL-base-fp16.param"
22-
sha256: "0254a056dce61b0c27dc9ec1b78b53bcf55315c540f55f051eb841aa992701ba"
23-
uri: "https://raw.githubusercontent.com/EdVince/Stable-Diffusion-NCNN/main/x86/linux/assets/AutoencoderKL-base-fp16.param"
24-
- filename: "stablediffusion_assets/AutoencoderKL-encoder-512-512-fp16.bin"
25-
sha256: "ddcb79a9951b9f91e05e087739ed69da2c1c4ae30ba4168cce350b49d617c9fa"
26-
uri: "https://github.com/EdVince/Stable-Diffusion-NCNN/releases/download/naifu/AutoencoderKL-encoder-512-512-fp16.bin"
27-
- filename: "stablediffusion_assets/AutoencoderKL-fp16.bin"
28-
sha256: "f02e71f80e70252734724bbfaed5c4ddd3a8ed7e61bb2175ff5f53099f0e35dd"
29-
uri: "https://github.com/EdVince/Stable-Diffusion-NCNN/releases/download/naifu/AutoencoderKL-fp16.bin"
30-
- filename: "stablediffusion_assets/FrozenCLIPEmbedder-fp16.bin"
31-
sha256: "1c9a12f4e1dd1b295a388045f7f28a2352a4d70c3dc96a542189a3dd7051fdd6"
32-
uri: "https://github.com/EdVince/Stable-Diffusion-NCNN/releases/download/naifu/FrozenCLIPEmbedder-fp16.bin"
33-
- filename: "stablediffusion_assets/FrozenCLIPEmbedder-fp16.param"
34-
sha256: "471afbe678dd1fd3fe764ef9c6eccaccb0a7d7e601f27b462aa926b20eb368c9"
35-
uri: "https://raw.githubusercontent.com/EdVince/Stable-Diffusion-NCNN/main/x86/linux/assets/FrozenCLIPEmbedder-fp16.param"
36-
- filename: "stablediffusion_assets/log_sigmas.bin"
37-
sha256: "a2089f8aa4c61f9c200feaec541ab3f5c94233b28deb6d5e8bcd974fa79b68ac"
38-
uri: "https://github.com/EdVince/Stable-Diffusion-NCNN/raw/main/x86/linux/assets/log_sigmas.bin"
39-
- filename: "stablediffusion_assets/UNetModel-256-256-MHA-fp16-opt.param"
40-
sha256: "a58c380229f09491776df837b7aa7adffc0a87821dc4708b34535da2e36e3da1"
41-
uri: "https://raw.githubusercontent.com/EdVince/Stable-Diffusion-NCNN/main/x86/linux/assets/UNetModel-256-256-MHA-fp16-opt.param"
42-
- filename: "stablediffusion_assets/UNetModel-512-512-MHA-fp16-opt.param"
43-
sha256: "f12034067062827bd7f43d1d21888d1f03905401acf6c6eea22be23c259636fa"
44-
uri: "https://raw.githubusercontent.com/EdVince/Stable-Diffusion-NCNN/main/x86/linux/assets/UNetModel-512-512-MHA-fp16-opt.param"
45-
- filename: "stablediffusion_assets/UNetModel-base-MHA-fp16.param"
46-
sha256: "696f6975de49f4325b53ce32aff81861a6d6c07cd9ce3f0aae2cc405350af38d"
47-
uri: "https://raw.githubusercontent.com/EdVince/Stable-Diffusion-NCNN/main/x86/linux/assets/UNetModel-base-MHA-fp16.param"
48-
- filename: "stablediffusion_assets/UNetModel-MHA-fp16.bin"
49-
sha256: "d618918d011bfc1f644c0f2a33bf84931bd53b28a98492b0a8ed6f3a818852c3"
50-
uri: "https://github.com/EdVince/Stable-Diffusion-NCNN/releases/download/naifu/UNetModel-MHA-fp16.bin"
51-
- filename: "stablediffusion_assets/vocab.txt"
52-
sha256: "e30e57b6f1e47616982ef898d8922be24e535b4fa3d0110477b3a6f02ebbae7d"
53-
uri: "https://raw.githubusercontent.com/EdVince/Stable-Diffusion-NCNN/main/x86/linux/assets/vocab.txt"
12+
- filename: "stable-diffusion-v1-5-pruned-emaonly-Q4_0.gguf"
13+
sha256: "b8944e9fe0b69b36ae1b5bb0185b3a7b8ef14347fe0fa9af6c64c4829022261f"
14+
uri: "huggingface://second-state/stable-diffusion-v1-5-GGUF/stable-diffusion-v1-5-pruned-emaonly-Q4_0.gguf"
5415

5516
usage: |
5617
curl http://localhost:8080/v1/images/generations \

‎backend/go/image/stablediffusion/main.go

-21
This file was deleted.

‎backend/go/image/stablediffusion/stablediffusion.go

-33
This file was deleted.

0 commit comments

Comments
 (0)