Skip to content

Commit 21bdfe5

Browse files
mudlermudler
authored
fix: use rice when embedding large binaries (#5309)
* fix(embed): use go-rice for large backend assets Golang embed FS has a hard limit that we might exceed when providing many binary alternatives. Signed-off-by: Ettore Di Giacinto <[email protected]> * simplify golang deps Signed-off-by: Ettore Di Giacinto <[email protected]> * chore(tests): switch to testcontainers and print logs Signed-off-by: Ettore Di Giacinto <[email protected]> * fix(tests): do not build a test binary Signed-off-by: Ettore Di Giacinto <[email protected]> * small fixup Signed-off-by: Ettore Di Giacinto <[email protected]> --------- Signed-off-by: Ettore Di Giacinto <[email protected]>
1 parent 7ebd7b2 commit 21bdfe5

File tree

14 files changed

+179
-121
lines changed

14 files changed

+179
-121
lines changed

‎.github/workflows/release.yaml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ jobs:
3636
sudo apt-get update
3737
sudo apt-get install build-essential ffmpeg protobuf-compiler ccache upx-ucl gawk
3838
sudo apt-get install -qy binutils-aarch64-linux-gnu gcc-aarch64-linux-gnu g++-aarch64-linux-gnu libgmock-dev
39+
make install-go-tools
3940
- name: Install CUDA Dependencies
4041
run: |
4142
curl -O https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/cross-linux-aarch64/cuda-keyring_1.1-1_all.deb
@@ -151,6 +152,7 @@ jobs:
151152
run: |
152153
sudo apt-get update
153154
sudo apt-get install -y wget curl build-essential ffmpeg protobuf-compiler ccache upx-ucl gawk cmake libgmock-dev
155+
make install-go-tools
154156
- name: Intel Dependencies
155157
run: |
156158
wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null
@@ -253,8 +255,7 @@ jobs:
253255
- name: Dependencies
254256
run: |
255257
brew install protobuf grpc
256-
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@8ba23be9613c672d40ae261d2a1335d639bdd59b
257-
go install google.golang.org/protobuf/cmd/[email protected]
258+
make install-go-tools
258259
- name: Build
259260
id: build
260261
run: |
@@ -295,8 +296,7 @@ jobs:
295296
- name: Dependencies
296297
run: |
297298
brew install protobuf grpc libomp llvm
298-
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af
299-
go install google.golang.org/protobuf/cmd/[email protected]
299+
make install-go-tools
300300
- name: Build
301301
id: build
302302
run: |

‎.github/workflows/test.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ jobs:
9696
9797
go install google.golang.org/protobuf/cmd/[email protected]
9898
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af
99+
go install github.com/GeertJohan/go.rice/rice@latest
99100
100101
# The python3-grpc-tools package in 22.04 is too old
101102
pip install --user grpcio-tools
@@ -183,6 +184,7 @@ jobs:
183184
rm protoc.zip
184185
go install google.golang.org/protobuf/cmd/[email protected]
185186
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af
187+
go install github.com/GeertJohan/go.rice/rice@latest
186188
PATH="$PATH:$HOME/go/bin" make protogen-go
187189
- name: Build images
188190
run: |
@@ -222,6 +224,7 @@ jobs:
222224
run: |
223225
brew install protobuf grpc make protoc-gen-go protoc-gen-go-grpc libomp llvm
224226
pip install --user --no-cache-dir grpcio-tools
227+
go install github.com/GeertJohan/go.rice/rice@latest
225228
- name: Test
226229
run: |
227230
export C_INCLUDE_PATH=/usr/local/include

‎Dockerfile

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,10 @@ EOT
4646
RUN curl -L -s https://go.dev/dl/go${GO_VERSION}.linux-${TARGETARCH}.tar.gz | tar -C /usr/local -xz
4747
ENV PATH=$PATH:/root/go/bin:/usr/local/go/bin
4848

49-
# Install grpc compilers
49+
# Install grpc compilers and rice
5050
RUN go install google.golang.org/protobuf/cmd/[email protected] && \
51-
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af
51+
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af && \
52+
go install github.com/GeertJohan/go.rice/rice@latest
5253

5354
COPY --chmod=644 custom-ca-certs/* /usr/local/share/ca-certificates/
5455
RUN update-ca-certificates
@@ -300,14 +301,7 @@ COPY .git .
300301
RUN make prepare
301302

302303
## Build the binary
303-
## If it's CUDA or hipblas, we want to skip some of the llama-compat backends to save space
304-
## We only leave the most CPU-optimized variant and the fallback for the cublas/hipblas build
305-
## (both will use CUDA or hipblas for the actual computation)
306-
RUN if [ "${BUILD_TYPE}" = "cublas" ] || [ "${BUILD_TYPE}" = "hipblas" ]; then \
307-
SKIP_GRPC_BACKEND="backend-assets/grpc/llama-cpp-avx512 backend-assets/grpc/llama-cpp-avx backend-assets/grpc/llama-cpp-avx2" make build; \
308-
else \
309-
make build; \
310-
fi
304+
RUN make build
311305

312306
RUN if [ ! -d "/build/sources/go-piper/piper-phonemize/pi/lib/" ]; then \
313307
mkdir -p /build/sources/go-piper/piper-phonemize/pi/lib/ \

‎Makefile

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -337,8 +337,14 @@ clean-tests:
337337
clean-dc: clean
338338
cp -r /build/backend-assets /workspace/backend-assets
339339

340+
## Install Go tools
341+
install-go-tools:
342+
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af
343+
go install google.golang.org/protobuf/cmd/[email protected]
344+
go install github.com/GeertJohan/go.rice/rice@latest
345+
340346
## Build:
341-
build: prepare backend-assets grpcs ## Build the project
347+
build: prepare backend-assets grpcs install-go-tools ## Build the project
342348
$(info ${GREEN}I local-ai build info:${RESET})
343349
$(info ${GREEN}I BUILD_TYPE: ${YELLOW}$(BUILD_TYPE)${RESET})
344350
$(info ${GREEN}I GO_TAGS: ${YELLOW}$(GO_TAGS)${RESET})
@@ -348,7 +354,9 @@ ifneq ($(BACKEND_LIBS),)
348354
$(MAKE) backend-assets/lib
349355
cp -f $(BACKEND_LIBS) backend-assets/lib/
350356
endif
357+
rm -rf $(BINARY_NAME) || true
351358
CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o $(BINARY_NAME) ./
359+
rice append --exec $(BINARY_NAME)
352360

353361
build-minimal:
354362
BUILD_GRPC_FOR_BACKEND_LLAMA=true GRPC_BACKENDS="backend-assets/grpc/llama-cpp-avx2" GO_TAGS=p2p $(MAKE) build
@@ -420,6 +428,7 @@ prepare-test: grpcs
420428
cp -rf backend-assets core/http
421429
cp tests/models_fixtures/* test-models
422430

431+
## Test targets
423432
test: prepare test-models/testmodel.ggml grpcs
424433
@echo 'Running tests'
425434
export GO_TAGS="tts debug"
@@ -494,7 +503,7 @@ protogen: protogen-go protogen-python
494503
protogen-clean: protogen-go-clean protogen-python-clean
495504

496505
.PHONY: protogen-go
497-
protogen-go:
506+
protogen-go: install-go-tools
498507
mkdir -p pkg/grpc/proto
499508
protoc --experimental_allow_proto3_optional -Ibackend/ --go_out=pkg/grpc/proto/ --go_opt=paths=source_relative --go-grpc_out=pkg/grpc/proto/ --go-grpc_opt=paths=source_relative \
500509
backend/backend.proto

‎assets.go

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,15 @@
11
package main
22

3-
import "embed"
3+
import (
4+
rice "github.com/GeertJohan/go.rice"
5+
)
46

5-
//go:embed backend-assets/*
6-
var backendAssets embed.FS
7+
var backendAssets *rice.Box
8+
9+
func init() {
10+
var err error
11+
backendAssets, err = rice.FindBox("backend-assets")
12+
if err != nil {
13+
panic(err)
14+
}
15+
}

‎core/cli/context/context.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
package cliContext
22

3-
import "embed"
3+
import (
4+
rice "github.com/GeertJohan/go.rice"
5+
)
46

57
type Context struct {
68
Debug bool `env:"LOCALAI_DEBUG,DEBUG" default:"false" hidden:"" help:"DEPRECATED, use --log-level=debug instead. Enable debug logging"`
79
LogLevel *string `env:"LOCALAI_LOG_LEVEL" enum:"error,warn,info,debug,trace" help:"Set the level of logs to output [${enum}]"`
810

911
// This field is not a command line argument/flag, the struct tag excludes it from the parsed CLI
10-
BackendAssets embed.FS `kong:"-"`
12+
BackendAssets *rice.Box `kong:"-"`
1113
}

‎core/config/application_config.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@ package config
22

33
import (
44
"context"
5-
"embed"
65
"encoding/json"
76
"regexp"
87
"time"
98

9+
rice "github.com/GeertJohan/go.rice"
1010
"github.com/mudler/LocalAI/pkg/xsysinfo"
1111
"github.com/rs/zerolog/log"
1212
)
@@ -47,7 +47,7 @@ type ApplicationConfig struct {
4747

4848
Galleries []Gallery
4949

50-
BackendAssets embed.FS
50+
BackendAssets *rice.Box
5151
AssetsDestination string
5252

5353
ExternalGRPCBackends map[string]string
@@ -198,7 +198,7 @@ func WithBackendAssetsOutput(out string) AppOption {
198198
}
199199
}
200200

201-
func WithBackendAssets(f embed.FS) AppOption {
201+
func WithBackendAssets(f *rice.Box) AppOption {
202202
return func(o *ApplicationConfig) {
203203
o.BackendAssets = f
204204
}

‎core/http/app_test.go

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package http_test
33
import (
44
"bytes"
55
"context"
6-
"embed"
76
"encoding/json"
87
"fmt"
98
"io"
@@ -24,6 +23,7 @@ import (
2423
. "github.com/onsi/gomega"
2524
"gopkg.in/yaml.v3"
2625

26+
rice "github.com/GeertJohan/go.rice"
2727
openaigo "github.com/otiai10/openaigo"
2828
"github.com/sashabaranov/go-openai"
2929
"github.com/sashabaranov/go-openai/jsonschema"
@@ -264,8 +264,15 @@ func getRequest(url string, header http.Header) (error, int, []byte) {
264264

265265
const bertEmbeddingsURL = `https://gist.githubusercontent.com/mudler/0a080b166b87640e8644b09c2aee6e3b/raw/f0e8c26bb72edc16d9fbafbfd6638072126ff225/bert-embeddings-gallery.yaml`
266266

267-
//go:embed backend-assets/*
268-
var backendAssets embed.FS
267+
var backendAssets *rice.Box
268+
269+
func init() {
270+
var err error
271+
backendAssets, err = rice.FindBox("backend-assets")
272+
if err != nil {
273+
panic(err)
274+
}
275+
}
269276

270277
var _ = Describe("API test", func() {
271278

‎go.mod

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ toolchain go1.23.1
66

77
require (
88
dario.cat/mergo v1.0.1
9+
github.com/GeertJohan/go.rice v1.0.3
910
github.com/Masterminds/sprig/v3 v3.3.0
1011
github.com/alecthomas/kong v0.9.0
1112
github.com/census-instrumentation/opencensus-proto v0.4.1
@@ -43,7 +44,6 @@ require (
4344
github.com/onsi/ginkgo/v2 v2.22.2
4445
github.com/onsi/gomega v1.36.2
4546
github.com/orcaman/writerseeker v0.0.0-20200621085525-1d3f536ff85e
46-
github.com/ory/dockertest/v3 v3.10.0
4747
github.com/otiai10/openaigo v1.7.0
4848
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
4949
github.com/prometheus/client_golang v1.20.5
@@ -55,7 +55,7 @@ require (
5555
github.com/streamer45/silero-vad-go v0.2.1
5656
github.com/stretchr/testify v1.10.0
5757
github.com/swaggo/swag v1.16.3
58-
github.com/thxcode/gguf-parser-go v0.1.0
58+
github.com/testcontainers/testcontainers-go v0.35.0
5959
github.com/tmc/langchaingo v0.1.12
6060
github.com/valyala/fasthttp v1.55.0
6161
go.opentelemetry.io/otel v1.34.0
@@ -75,19 +75,26 @@ require (
7575
cloud.google.com/go/auth v0.4.1 // indirect
7676
cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect
7777
cloud.google.com/go/compute/metadata v0.5.0 // indirect
78+
github.com/containerd/platforms v0.2.1 // indirect
79+
github.com/cpuguy83/dockercfg v0.3.2 // indirect
80+
github.com/daaku/go.zipexe v1.0.2 // indirect
81+
github.com/distribution/reference v0.6.0 // indirect
7882
github.com/dustin/go-humanize v1.0.1 // indirect
7983
github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect
8084
github.com/fasthttp/websocket v1.5.3 // indirect
8185
github.com/felixge/httpsnoop v1.0.4 // indirect
8286
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
83-
github.com/go-viper/mapstructure/v2 v2.0.0 // indirect
8487
github.com/google/s2a-go v0.1.7 // indirect
8588
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
8689
github.com/googleapis/gax-go/v2 v2.12.4 // indirect
8790
github.com/json-iterator/go v1.1.12 // indirect
91+
github.com/magiconair/properties v1.8.7 // indirect
8892
github.com/moby/docker-image-spec v1.3.1 // indirect
93+
github.com/moby/patternmatcher v0.6.0 // indirect
94+
github.com/moby/sys/user v0.1.0 // indirect
8995
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
9096
github.com/modern-go/reflect2 v1.0.2 // indirect
97+
github.com/morikuni/aec v1.0.0 // indirect
9198
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
9299
github.com/pion/datachannel v1.5.10 // indirect
93100
github.com/pion/dtls/v2 v2.2.12 // indirect
@@ -130,7 +137,6 @@ require (
130137
github.com/Masterminds/semver/v3 v3.3.0 // indirect
131138
github.com/Microsoft/go-winio v0.6.2 // indirect
132139
github.com/Microsoft/hcsshim v0.11.7 // indirect
133-
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
134140
github.com/StackExchange/wmi v1.2.1 // indirect
135141
github.com/alecthomas/chroma/v2 v2.8.0 // indirect
136142
github.com/andybalholm/brotli v1.1.0 // indirect
@@ -154,7 +160,7 @@ require (
154160
github.com/dlclark/regexp2 v1.10.0 // indirect
155161
github.com/docker/cli v27.0.3+incompatible // indirect
156162
github.com/docker/distribution v2.8.2+incompatible // indirect
157-
github.com/docker/docker v27.0.3+incompatible
163+
github.com/docker/docker v27.1.1+incompatible
158164
github.com/docker/docker-credential-helpers v0.7.0 // indirect
159165
github.com/docker/go-connections v0.5.0 // indirect
160166
github.com/docker/go-units v0.5.0 // indirect
@@ -183,7 +189,6 @@ require (
183189
github.com/google/go-cmp v0.6.0 // indirect
184190
github.com/google/gopacket v1.1.19 // indirect
185191
github.com/google/pprof v0.0.0-20250208200701-d0013a598941 // indirect
186-
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
187192
github.com/gorilla/css v1.0.1 // indirect
188193
github.com/gorilla/websocket v1.5.3 // indirect
189194
github.com/hashicorp/errwrap v1.1.0 // indirect
@@ -257,7 +262,6 @@ require (
257262
github.com/olekukonko/tablewriter v0.0.5 // indirect
258263
github.com/opencontainers/go-digest v1.0.0 // indirect
259264
github.com/opencontainers/image-spec v1.1.0
260-
github.com/opencontainers/runc v1.1.12 // indirect
261265
github.com/opencontainers/runtime-spec v1.2.0 // indirect
262266
github.com/opentracing/opentracing-go v1.2.0 // indirect
263267
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
@@ -295,9 +299,6 @@ require (
295299
github.com/vishvananda/netlink v1.3.0 // indirect
296300
github.com/vishvananda/netns v0.0.5 // indirect
297301
github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
298-
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
299-
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
300-
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
301302
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
302303
github.com/yuin/goldmark v1.5.4 // indirect
303304
github.com/yuin/goldmark-emoji v1.0.2 // indirect

0 commit comments

Comments
 (0)