Skip to content

Commit d9c2bb8

Browse files
authored
Merge branch 'mudler:master' into mlx_cache
2 parents 1585ddd + 3013d1c commit d9c2bb8

File tree

12 files changed

+752
-95
lines changed

12 files changed

+752
-95
lines changed

backend/go/stablediffusion-ggml/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
88

99
# stablediffusion.cpp (ggml)
1010
STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp
11-
STABLEDIFFUSION_GGML_VERSION?=11ab095230b2b67210f5da4d901588d56c71fe3a
11+
STABLEDIFFUSION_GGML_VERSION?=43a70e819b9254dee0d017305d6992f6bb27f850
1212

1313
CMAKE_ARGS+=-DGGML_MAX_NAME=128
1414

backend/go/whisper/Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
88

99
# whisper.cpp version
1010
WHISPER_REPO?=https://github.com/ggml-org/whisper.cpp
11-
WHISPER_CPP_VERSION?=9f5ed26e43c680bece09df7bdc8c1b7835f0e537
11+
WHISPER_CPP_VERSION?=2551e4ce98db69027d08bd99bcc3f1a4e2ad2cef
1212
SO_TARGET?=libgowhisper.so
1313

1414
CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

core/application/startup.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,9 @@ import (
1111
"github.com/mudler/LocalAI/core/config"
1212
"github.com/mudler/LocalAI/core/gallery"
1313
"github.com/mudler/LocalAI/core/services"
14+
coreStartup "github.com/mudler/LocalAI/core/startup"
1415
"github.com/mudler/LocalAI/internal"
1516

16-
coreStartup "github.com/mudler/LocalAI/core/startup"
1717
"github.com/mudler/LocalAI/pkg/model"
1818
"github.com/mudler/LocalAI/pkg/xsysinfo"
1919
"github.com/rs/zerolog/log"
@@ -75,7 +75,7 @@ func New(opts ...config.AppOption) (*Application, error) {
7575
}
7676

7777
for _, backend := range options.ExternalBackends {
78-
if err := coreStartup.InstallExternalBackends(options.Context, options.BackendGalleries, options.SystemState, application.ModelLoader(), nil, backend, "", ""); err != nil {
78+
if err := services.InstallExternalBackend(options.Context, options.BackendGalleries, options.SystemState, application.ModelLoader(), nil, backend, "", ""); err != nil {
7979
log.Error().Err(err).Msg("error installing external backend")
8080
}
8181
}

core/cli/backends.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,11 @@ import (
77

88
cliContext "github.com/mudler/LocalAI/core/cli/context"
99
"github.com/mudler/LocalAI/core/config"
10+
"github.com/mudler/LocalAI/core/gallery"
11+
"github.com/mudler/LocalAI/core/services"
1012
"github.com/mudler/LocalAI/pkg/model"
1113
"github.com/mudler/LocalAI/pkg/system"
1214

13-
"github.com/mudler/LocalAI/core/gallery"
14-
"github.com/mudler/LocalAI/core/startup"
1515
"github.com/rs/zerolog/log"
1616
"github.com/schollz/progressbar/v3"
1717
)
@@ -103,7 +103,7 @@ func (bi *BackendsInstall) Run(ctx *cliContext.Context) error {
103103
}
104104

105105
modelLoader := model.NewModelLoader(systemState)
106-
err = startup.InstallExternalBackends(context.Background(), galleries, systemState, modelLoader, progressCallback, bi.BackendArgs, bi.Name, bi.Alias)
106+
err = services.InstallExternalBackend(context.Background(), galleries, systemState, modelLoader, progressCallback, bi.BackendArgs, bi.Name, bi.Alias)
107107
if err != nil {
108108
return err
109109
}

core/http/routes/ui_api.go

Lines changed: 77 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -81,13 +81,17 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
8181
}
8282

8383
// Determine if it's a model or backend
84-
isBackend := false
85-
backends, _ := gallery.AvailableBackends(appConfig.BackendGalleries, appConfig.SystemState)
86-
for _, b := range backends {
87-
backendID := fmt.Sprintf("%s@%s", b.Gallery.Name, b.Name)
88-
if backendID == galleryID || b.Name == galleryID {
89-
isBackend = true
90-
break
84+
// First check if it was explicitly marked as a backend operation
85+
isBackend := opcache.IsBackendOp(galleryID)
86+
// If not explicitly marked, check if it matches a known backend from the gallery
87+
if !isBackend {
88+
backends, _ := gallery.AvailableBackends(appConfig.BackendGalleries, appConfig.SystemState)
89+
for _, b := range backends {
90+
backendID := fmt.Sprintf("%s@%s", b.Gallery.Name, b.Name)
91+
if backendID == galleryID || b.Name == galleryID {
92+
isBackend = true
93+
break
94+
}
9195
}
9296
}
9397

@@ -645,7 +649,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
645649
}
646650

647651
uid := id.String()
648-
opcache.Set(backendID, uid)
652+
opcache.SetBackend(backendID, uid)
649653

650654
ctx, cancelFunc := context.WithCancel(context.Background())
651655
op := services.GalleryOp[gallery.GalleryBackend, any]{
@@ -667,6 +671,70 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
667671
})
668672
})
669673

674+
// Install backend from external source (OCI image, URL, or path)
675+
app.POST("/api/backends/install-external", func(c echo.Context) error {
676+
// Request body structure
677+
type ExternalBackendRequest struct {
678+
URI string `json:"uri"`
679+
Name string `json:"name"`
680+
Alias string `json:"alias"`
681+
}
682+
683+
var req ExternalBackendRequest
684+
if err := c.Bind(&req); err != nil {
685+
return c.JSON(http.StatusBadRequest, map[string]interface{}{
686+
"error": "invalid request body",
687+
})
688+
}
689+
690+
// Validate required fields
691+
if req.URI == "" {
692+
return c.JSON(http.StatusBadRequest, map[string]interface{}{
693+
"error": "uri is required",
694+
})
695+
}
696+
697+
log.Debug().Str("uri", req.URI).Str("name", req.Name).Str("alias", req.Alias).Msg("API job submitted to install external backend")
698+
699+
id, err := uuid.NewUUID()
700+
if err != nil {
701+
return c.JSON(http.StatusInternalServerError, map[string]interface{}{
702+
"error": err.Error(),
703+
})
704+
}
705+
706+
uid := id.String()
707+
708+
// Use URI as the key for opcache, or name if provided
709+
cacheKey := req.URI
710+
if req.Name != "" {
711+
cacheKey = req.Name
712+
}
713+
opcache.SetBackend(cacheKey, uid)
714+
715+
ctx, cancelFunc := context.WithCancel(context.Background())
716+
op := services.GalleryOp[gallery.GalleryBackend, any]{
717+
ID: uid,
718+
GalleryElementName: req.Name, // May be empty, will be derived during installation
719+
Galleries: appConfig.BackendGalleries,
720+
Context: ctx,
721+
CancelFunc: cancelFunc,
722+
ExternalURI: req.URI,
723+
ExternalName: req.Name,
724+
ExternalAlias: req.Alias,
725+
}
726+
// Store cancellation function immediately so queued operations can be cancelled
727+
galleryService.StoreCancellation(uid, cancelFunc)
728+
go func() {
729+
galleryService.BackendGalleryChannel <- op
730+
}()
731+
732+
return c.JSON(200, map[string]interface{}{
733+
"jobID": uid,
734+
"message": "External backend installation started",
735+
})
736+
})
737+
670738
app.POST("/api/backends/delete/:id", func(c echo.Context) error {
671739
backendID := c.Param("id")
672740
// URL decode the backend ID
@@ -692,7 +760,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
692760

693761
uid := id.String()
694762

695-
opcache.Set(backendID, uid)
763+
opcache.SetBackend(backendID, uid)
696764

697765
ctx, cancelFunc := context.WithCancel(context.Background())
698766
op := services.GalleryOp[gallery.GalleryBackend, any]{
Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
package routes_test
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"encoding/json"
7+
"io"
8+
"net/http"
9+
"net/http/httptest"
10+
"os"
11+
"path/filepath"
12+
"testing"
13+
14+
"github.com/labstack/echo/v4"
15+
"github.com/mudler/LocalAI/core/config"
16+
"github.com/mudler/LocalAI/core/gallery"
17+
"github.com/mudler/LocalAI/core/http/routes"
18+
"github.com/mudler/LocalAI/core/services"
19+
"github.com/mudler/LocalAI/pkg/model"
20+
"github.com/mudler/LocalAI/pkg/system"
21+
. "github.com/onsi/ginkgo/v2"
22+
. "github.com/onsi/gomega"
23+
)
24+
25+
func TestRoutes(t *testing.T) {
26+
RegisterFailHandler(Fail)
27+
RunSpecs(t, "Routes Suite")
28+
}
29+
30+
var _ = Describe("Backend API Routes", func() {
31+
var (
32+
app *echo.Echo
33+
tempDir string
34+
appConfig *config.ApplicationConfig
35+
galleryService *services.GalleryService
36+
modelLoader *model.ModelLoader
37+
systemState *system.SystemState
38+
configLoader *config.ModelConfigLoader
39+
)
40+
41+
BeforeEach(func() {
42+
var err error
43+
tempDir, err = os.MkdirTemp("", "backend-routes-test-*")
44+
Expect(err).NotTo(HaveOccurred())
45+
46+
systemState, err = system.GetSystemState(
47+
system.WithBackendPath(filepath.Join(tempDir, "backends")),
48+
)
49+
Expect(err).NotTo(HaveOccurred())
50+
systemState.Model.ModelsPath = filepath.Join(tempDir, "models")
51+
52+
// Create directories
53+
err = os.MkdirAll(systemState.Backend.BackendsPath, 0750)
54+
Expect(err).NotTo(HaveOccurred())
55+
err = os.MkdirAll(systemState.Model.ModelsPath, 0750)
56+
Expect(err).NotTo(HaveOccurred())
57+
58+
modelLoader = model.NewModelLoader(systemState)
59+
configLoader = config.NewModelConfigLoader(tempDir)
60+
61+
appConfig = config.NewApplicationConfig(
62+
config.WithContext(context.Background()),
63+
)
64+
appConfig.SystemState = systemState
65+
appConfig.BackendGalleries = []config.Gallery{}
66+
67+
galleryService = services.NewGalleryService(appConfig, modelLoader)
68+
// Start the gallery service
69+
err = galleryService.Start(context.Background(), configLoader, systemState)
70+
Expect(err).NotTo(HaveOccurred())
71+
72+
app = echo.New()
73+
74+
// Register the API routes for backends
75+
opcache := services.NewOpCache(galleryService)
76+
routes.RegisterUIAPIRoutes(app, configLoader, modelLoader, appConfig, galleryService, opcache, nil)
77+
})
78+
79+
AfterEach(func() {
80+
os.RemoveAll(tempDir)
81+
})
82+
83+
Describe("POST /api/backends/install-external", func() {
84+
It("should return error when URI is missing", func() {
85+
reqBody := map[string]string{
86+
"name": "test-backend",
87+
}
88+
jsonBody, err := json.Marshal(reqBody)
89+
Expect(err).NotTo(HaveOccurred())
90+
91+
req := httptest.NewRequest(http.MethodPost, "/api/backends/install-external", bytes.NewBuffer(jsonBody))
92+
req.Header.Set("Content-Type", "application/json")
93+
rec := httptest.NewRecorder()
94+
95+
app.ServeHTTP(rec, req)
96+
97+
Expect(rec.Code).To(Equal(http.StatusBadRequest))
98+
99+
var response map[string]interface{}
100+
err = json.Unmarshal(rec.Body.Bytes(), &response)
101+
Expect(err).NotTo(HaveOccurred())
102+
Expect(response["error"]).To(Equal("uri is required"))
103+
})
104+
105+
It("should accept valid request and return job ID", func() {
106+
reqBody := map[string]string{
107+
"uri": "oci://quay.io/example/backend:latest",
108+
"name": "test-backend",
109+
"alias": "test-alias",
110+
}
111+
jsonBody, err := json.Marshal(reqBody)
112+
Expect(err).NotTo(HaveOccurred())
113+
114+
req := httptest.NewRequest(http.MethodPost, "/api/backends/install-external", bytes.NewBuffer(jsonBody))
115+
req.Header.Set("Content-Type", "application/json")
116+
rec := httptest.NewRecorder()
117+
118+
app.ServeHTTP(rec, req)
119+
120+
Expect(rec.Code).To(Equal(http.StatusOK))
121+
122+
var response map[string]interface{}
123+
err = json.Unmarshal(rec.Body.Bytes(), &response)
124+
Expect(err).NotTo(HaveOccurred())
125+
Expect(response["jobID"]).NotTo(BeEmpty())
126+
Expect(response["message"]).To(Equal("External backend installation started"))
127+
})
128+
129+
It("should accept request with only URI", func() {
130+
reqBody := map[string]string{
131+
"uri": "/path/to/local/backend",
132+
}
133+
jsonBody, err := json.Marshal(reqBody)
134+
Expect(err).NotTo(HaveOccurred())
135+
136+
req := httptest.NewRequest(http.MethodPost, "/api/backends/install-external", bytes.NewBuffer(jsonBody))
137+
req.Header.Set("Content-Type", "application/json")
138+
rec := httptest.NewRecorder()
139+
140+
app.ServeHTTP(rec, req)
141+
142+
Expect(rec.Code).To(Equal(http.StatusOK))
143+
144+
var response map[string]interface{}
145+
err = json.Unmarshal(rec.Body.Bytes(), &response)
146+
Expect(err).NotTo(HaveOccurred())
147+
Expect(response["jobID"]).NotTo(BeEmpty())
148+
})
149+
150+
It("should return error for invalid JSON body", func() {
151+
req := httptest.NewRequest(http.MethodPost, "/api/backends/install-external", bytes.NewBufferString("invalid json"))
152+
req.Header.Set("Content-Type", "application/json")
153+
rec := httptest.NewRecorder()
154+
155+
app.ServeHTTP(rec, req)
156+
157+
Expect(rec.Code).To(Equal(http.StatusBadRequest))
158+
})
159+
})
160+
161+
Describe("GET /api/backends/job/:uid", func() {
162+
It("should return queued status for unknown job", func() {
163+
req := httptest.NewRequest(http.MethodGet, "/api/backends/job/unknown-job-id", nil)
164+
rec := httptest.NewRecorder()
165+
166+
app.ServeHTTP(rec, req)
167+
168+
Expect(rec.Code).To(Equal(http.StatusOK))
169+
170+
var response map[string]interface{}
171+
err := json.Unmarshal(rec.Body.Bytes(), &response)
172+
Expect(err).NotTo(HaveOccurred())
173+
Expect(response["queued"]).To(Equal(true))
174+
Expect(response["processed"]).To(Equal(false))
175+
})
176+
})
177+
})
178+
179+
// Helper function to make POST request
180+
func postRequest(url string, body interface{}) (*http.Response, error) {
181+
jsonBody, err := json.Marshal(body)
182+
if err != nil {
183+
return nil, err
184+
}
185+
186+
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonBody))
187+
if err != nil {
188+
return nil, err
189+
}
190+
req.Header.Set("Content-Type", "application/json")
191+
192+
client := &http.Client{}
193+
return client.Do(req)
194+
}
195+
196+
// Helper function to read response body
197+
func readResponseBody(resp *http.Response) (map[string]interface{}, error) {
198+
defer resp.Body.Close()
199+
body, err := io.ReadAll(resp.Body)
200+
if err != nil {
201+
return nil, err
202+
}
203+
204+
var result map[string]interface{}
205+
err = json.Unmarshal(body, &result)
206+
return result, err
207+
}
208+
209+
// Avoid unused import errors
210+
var _ = gallery.GalleryModel{}

0 commit comments

Comments
 (0)