diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d0b7c61 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,40 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.25.6' + + + - name: Run go vet + run: go vet ./... + + - name: Run tests + run: go test ./... + + build: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.25.6' + + + - name: Build + run: go build ./cmd/smart-llama/ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..1edd636 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,85 @@ +name: Release + +on: + push: + tags: + - 'v*.*.*' + +permissions: + contents: write + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - goos: linux + goarch: amd64 + suffix: linux-amd64 + - goos: linux + goarch: arm64 + suffix: linux-arm64 + - goos: darwin + goarch: amd64 + suffix: darwin-amd64 + - goos: darwin + goarch: arm64 + suffix: darwin-arm64 + - goos: windows + goarch: amd64 + suffix: windows-amd64 + extension: .exe + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.25.6' + + + - name: Run tests + run: go test ./... + + - name: Build binary + env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + CGO_ENABLED: 0 + run: | + go build -ldflags="-s -w" -o smart-llama-${{ matrix.suffix }}${{ matrix.extension }} ./cmd/smart-llama/ + + - name: Generate checksum + run: | + sha256sum smart-llama-${{ matrix.suffix }}${{ matrix.extension }} > smart-llama-${{ matrix.suffix }}${{ matrix.extension }}.sha256 + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: smart-llama-${{ matrix.suffix }} + path: | + smart-llama-${{ matrix.suffix }}${{ matrix.extension }} + smart-llama-${{ matrix.suffix }}${{ matrix.extension }}.sha256 + + release: + needs: build + runs-on: ubuntu-latest + steps: + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + merge-multiple: true + + - name: List artifacts + run: ls -la artifacts/ + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + generate_release_notes: true + files: | + artifacts/* diff --git a/.gitignore b/.gitignore index aaadf73..1301165 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ *.dll *.so *.dylib +smart-llama # Test binary, built with `go test -c` *.test diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..1bd9e6f --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,197 @@ +# AGENTS.md - Coding Agent Guidelines for smart-llama + +This document provides guidance for AI coding agents working in this repository. + +## Project Overview + +**smart-llama** is a Go server that wraps `llama-server` (llama.cpp) as a subprocess +with dynamic model loading and an OpenAI-compatible API. + +## Build & Test Commands + +### Build +```bash +# Build the binary +go build ./cmd/smart-llama/ + +# Build with optimizations (for release) +CGO_ENABLED=0 go build -ldflags="-s -w" ./cmd/smart-llama/ +``` + +### Test +```bash +# Run all tests +go test ./... + +# Run tests with verbose output +go test -v ./... + +# Run a single test function +go test -v -run TestLoadConfig ./internal/config/ + +# Run a specific subtest +go test -v -run "TestLoadConfig/valid_config_file" ./internal/config/ + +# Run tests for a specific package +go test ./internal/process/ + +# Run tests with coverage +go test -cover ./... +``` + +### Lint +```bash +# Run go vet (always do this before committing) +go vet ./... + +# Format code +go fmt ./... +``` + +## Project Structure + +``` +cmd/smart-llama/ # Application entry point +internal/ + config/ # YAML configuration loading + process/ # llama-server subprocess management + proxy/ # HTTP reverse proxy to llama-server + server/ # HTTP server and routing +models/ # Model configuration files (*.yaml) +.github/workflows/ # CI and release workflows +``` + +## Code Style Guidelines + +### Imports + +Order imports in three groups, separated by blank lines: +1. Standard library +2. External dependencies +3. Internal packages + +```go +import ( + "context" + "fmt" + "net/http" + + "gopkg.in/yaml.v3" + + "github.com/fvigneault/smart-llama/internal/config" +) +``` + +### Naming Conventions + +- **Packages**: lowercase, single word (`config`, `process`, `proxy`) +- **Exported types/functions**: PascalCase (`NewManager`, `LoadConfig`) +- **Unexported types/functions**: camelCase (`loadModel`, `killProcessLocked`) +- **Interfaces**: describe behavior, often end in `-er` (`processManager`) +- **Constants**: PascalCase for exported, camelCase for unexported + +### Error Handling + +- Always wrap errors with context using `fmt.Errorf("description: %w", err)` +- Return early on errors +- Check errors explicitly, never ignore them + +```go +data, err := os.ReadFile(path) +if err != nil { + return nil, fmt.Errorf("failed to read config file: %w", err) +} +``` + +### Struct Tags + +Use consistent YAML tags for configuration structs: +```go +type Config struct { + Server ServerConfig `yaml:"server"` + DefaultModel string `yaml:"default_model"` +} +``` + +### Comments + +- Document all exported types and functions +- Use complete sentences starting with the function/type name +- No inline comments unless absolutely necessary + +```go +// NewManager creates a new process manager. +func NewManager(...) *Manager { +``` + +### Concurrency + +- Use `sync.RWMutex` for shared state +- Name lock-holding methods with `Locked` suffix (`killProcessLocked`) +- Always use `defer` for unlocking + +```go +func (m *Manager) CurrentModel() string { + m.mu.RLock() + defer m.mu.RUnlock() + return m.currentModelValue +} +``` + +## Testing Guidelines + +### Test File Naming +- Test files: `*_test.go` in same package +- Use table-driven tests with subtests + +### Test Structure +```go +func TestFunctionName(t *testing.T) { + t.Run("descriptive scenario name", func(t *testing.T) { + // Arrange + // Act + // Assert + }) +} +``` + +### Mocking +- Define interfaces for dependencies (see `processManager` in server.go) +- Create mock implementations in test files +- Use `httptest.NewServer` for HTTP testing + +## Configuration + +### Global Config (`config.yaml`) +```yaml +server: + listen_addr: ":8080" + llama_server_port: 8081 +paths: + llama_server: "/usr/local/bin/llama-server" + models_dir: "./models" +default_model: "model-name" +``` + +### Model Config (`models/*.yaml`) +```yaml +name: model-name +model_path: /path/to/model.gguf +args: + - --ctx-size + - "8192" +``` + +## Git Workflow + +- Create feature branches: `feature/description` +- Use conventional commits: `feat:`, `fix:`, `docs:`, `test:`, `refactor:` +- Run `go vet ./...` and `go test ./...` before committing +- PR to `main` branch, never commit directly to `main` + +## CI/CD + +- **CI** (`.github/workflows/ci.yml`): Runs on PRs - `go vet` + `go test` +- **Release** (`.github/workflows/release.yml`): Runs on tags `v*.*.*` + - Builds binaries for Linux/macOS/Windows (amd64/arm64) + - Creates GitHub Release with checksums diff --git a/README.md b/README.md new file mode 100644 index 0000000..8309f5e --- /dev/null +++ b/README.md @@ -0,0 +1,116 @@ +# smart-llama + +A lightweight Go server that wraps `llama-server` (from llama.cpp) with dynamic model loading and an OpenAI-compatible API. + +## Why smart-llama? + +**The problem**: Running multiple LLM models locally requires either loading them all into memory (expensive) or manually restarting servers when switching models. + +**The solution**: smart-llama keeps one model loaded at a time and automatically swaps models when you request a different one. Think of it as a simpler Ollama that preserves full `llama-server` configuration flexibility. + +``` +Client -> smart-llama (:8080) -> llama-server subprocess (:8081) +``` + +**Key features**: +- Single model in memory at a time (automatic swap on request) +- Full control over `llama-server` arguments per model +- OpenAI-compatible API for easy integration +- YAML configuration (no CLI flags to remember) + +## Quick Start + +### 1. Install llama.cpp + +Build or install `llama-server` from [llama.cpp](https://github.com/ggerganov/llama.cpp). + +### 2. Configure + +**Server configuration** (`config.yaml`): + +```yaml +server: + listen_addr: ":8080" + llama_server_port: 8081 + +paths: + llama_server: "/usr/local/bin/llama-server" + models_dir: "./models" + +default_model: "llama3-8b" +``` + +**Model configuration** (`models/llama3-8b.yaml`): + +```yaml +name: llama3-8b +model_path: /path/to/llama-3-8b.gguf + +args: + - --ctx-size + - "8192" + - --n-gpu-layers + - "99" + - --flash-attn + - "on" +``` + +The `args` list accepts any `llama-server` argument directly. + +### 3. Run + +```bash +./smart-llama +``` + +### 4. Use + +```bash +# List available models +curl http://localhost:8080/v1/models + +# Chat completion +curl http://localhost:8080/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "llama3-8b", + "messages": [{"role": "user", "content": "Hello!"}] + }' + +# Health check +curl http://localhost:8080/health +``` + +## Configuration Reference + +### Server (`config.yaml`) + +| Field | Description | Default | +|-------|-------------|---------| +| `server.listen_addr` | Address to listen on | `:8080` | +| `server.llama_server_port` | Port for llama-server subprocess | `8081` | +| `paths.llama_server` | Path to llama-server binary | `llama-server` | +| `paths.models_dir` | Directory containing model configs | `./models` | +| `default_model` | Model to load on startup | (required) | +| `timeouts.startup` | Model startup timeout (seconds) | `120` | +| `timeouts.shutdown` | Graceful shutdown timeout (seconds) | `10` | + +### Model (`models/*.yaml`) + +| Field | Description | +|-------|-------------| +| `name` | Model identifier (used in API requests) | +| `model_path` | Absolute path to the `.gguf` file | +| `args` | List of `llama-server` arguments | + +## Build + +```bash +go build ./cmd/smart-llama/ +``` + +## Test + +```bash +go test ./... +``` diff --git a/cmd/smart-llama/main.go b/cmd/smart-llama/main.go new file mode 100644 index 0000000..d48a25c --- /dev/null +++ b/cmd/smart-llama/main.go @@ -0,0 +1,109 @@ +package main + +import ( + "fmt" + "log" + "log/slog" + "net/http" + "os" + "os/signal" + "syscall" + + "github.com/fvigneault/smart-llama/internal/config" + "github.com/fvigneault/smart-llama/internal/process" + "github.com/fvigneault/smart-llama/internal/proxy" + "github.com/fvigneault/smart-llama/internal/server" +) + +type App struct { + config *config.Config + log *slog.Logger + models map[string]*config.ModelConfig +} + +func NewApp() *App { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + return &App{ + log: logger, + } +} + +func (a *App) loadConfig() error { + loader := config.NewLoader() + cfg, err := loader.LoadConfig("config.yaml") + if err != nil { + return err + } + + models, err := loader.LoadModels(cfg.Paths.ModelsDir) + if err != nil { + return err + } + + a.config = cfg + a.models = models + + return nil +} + +func (a *App) run() error { + a.log.Info("starting smart-llama") + + if a.config.DefaultModel == "" { + a.log.Warn("no default model configured") + } + + processMgr := process.NewManager( + a.config.Paths.LlamaServer, + a.config.Server.LlamaServerPort, + a.config.Timeouts, + a.log, + ) + + proxyTarget := fmt.Sprintf("localhost:%d", a.config.Server.LlamaServerPort) + proxyHandler := proxy.NewHandler(proxyTarget) + + srv := server.NewServer( + a.config.Server.ListenAddr, + a.models, + processMgr, + proxyHandler, + a.log, + ) + + http.HandleFunc("/health", srv.HandleHealth) + http.HandleFunc("/v1/models", srv.HandleModelsList) + + shutdown := make(chan os.Signal, 1) + signal.Notify(shutdown, syscall.SIGINT, syscall.SIGTERM) + + go func() { + a.log.Info("server listening", "addr", a.config.Server.ListenAddr) + if err := http.ListenAndServe(a.config.Server.ListenAddr, nil); err != nil { + a.log.Error("server error", "error", err) + shutdown <- syscall.SIGTERM + } + }() + + <-shutdown + a.log.Info("shutting down") + + if processMgr.IsRunning() { + a.log.Info("stopping llama-server process") + processMgr.Stop() + } + + a.log.Info("shutdown complete") + return nil +} + +func main() { + app := NewApp() + if err := app.loadConfig(); err != nil { + log.Fatal(err) + } + + if err := app.run(); err != nil { + log.Fatal(err) + } +} diff --git a/cmd/smart-llama/main_test.go b/cmd/smart-llama/main_test.go new file mode 100644 index 0000000..d98129b --- /dev/null +++ b/cmd/smart-llama/main_test.go @@ -0,0 +1,13 @@ +package main + +import ( + "testing" +) + +func TestNewApp(t *testing.T) { + app := NewApp() + + if app == nil { + t.Fatal("NewApp() returned nil") + } +} diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..24de7ba --- /dev/null +++ b/config.yaml @@ -0,0 +1,14 @@ +server: + listen_addr: ":8080" + llama_server_port: 8081 + +paths: + llama_server: "/usr/local/bin/llama-server" + models_dir: "./models" + +default_model: "llama3-8b" + +timeouts: + shutdown: 10 + startup: 120 + health_check_interval: 1000 \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..204f6d7 --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module github.com/fvigneault/smart-llama + +go 1.25.0 + +require gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..4bc0337 --- /dev/null +++ b/go.sum @@ -0,0 +1,3 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..99fa515 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,186 @@ +package config + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "gopkg.in/yaml.v3" +) + +// Config represents the global application configuration. +type Config struct { + Server ServerConfig `yaml:"server"` + Paths PathsConfig `yaml:"paths"` + DefaultModel string `yaml:"default_model"` + Timeouts TimeoutsConfig `yaml:"timeouts"` +} + +// ServerConfig contains HTTP server settings. +type ServerConfig struct { + ListenAddr string `yaml:"listen_addr"` + LlamaServerPort int `yaml:"llama_server_port"` +} + +// PathsConfig contains file system paths. +type PathsConfig struct { + LlamaServer string `yaml:"llama_server"` + ModelsDir string `yaml:"models_dir"` +} + +// TimeoutsConfig contains timeout durations. +type TimeoutsConfig struct { + ShutdownSec int `yaml:"shutdown"` + StartupSec int `yaml:"startup"` + HealthCheckIntervalMs int `yaml:"health_check_interval"` +} + +// ShutdownDuration returns the shutdown timeout as a time.Duration. +func (t TimeoutsConfig) ShutdownDuration() time.Duration { + return time.Duration(t.ShutdownSec) * time.Second +} + +// StartupDuration returns the startup timeout as a time.Duration. +func (t TimeoutsConfig) StartupDuration() time.Duration { + return time.Duration(t.StartupSec) * time.Second +} + +// HealthCheckInterval returns the health check interval as a time.Duration. +func (t TimeoutsConfig) HealthCheckInterval() time.Duration { + return time.Duration(t.HealthCheckIntervalMs) * time.Millisecond +} + +// ModelConfig represents configuration for a single model. +type ModelConfig struct { + Name string `yaml:"name"` + ModelPath string `yaml:"model_path"` + Args []string `yaml:"args"` +} + +// BuildArgs constructs the full command line arguments for llama-server. +func (m ModelConfig) BuildArgs() []string { + args := make([]string, 0, len(m.Args)+2) + args = append(args, "-m", m.ModelPath) + args = append(args, m.Args...) + return args +} + +// Loader handles configuration loading. +type Loader struct{} + +// NewLoader creates a new configuration loader. +func NewLoader() *Loader { + return &Loader{} +} + +// LoadConfig loads the global configuration from a file path. +func (l *Loader) LoadConfig(path string) (*Config, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read config file: %w", err) + } + + config := DefaultConfig() + + if err := yaml.Unmarshal(data, &config); err != nil { + return nil, fmt.Errorf("failed to parse config: %w", err) + } + + return &config, nil +} + +// LoadModels loads all model configurations from the models directory. +func (l *Loader) LoadModels(modelsDir string) (map[string]*ModelConfig, error) { + entries, err := os.ReadDir(modelsDir) + if err != nil { + return nil, fmt.Errorf("failed to read models directory: %w", err) + } + + models := make(map[string]*ModelConfig) + + for _, entry := range entries { + if entry.IsDir() { + continue + } + + ext := filepath.Ext(entry.Name()) + if ext != ".yaml" && ext != ".yml" { + continue + } + + path := filepath.Join(modelsDir, entry.Name()) + model, err := l.loadModel(path) + if err != nil { + return nil, fmt.Errorf("failed to load model from %s: %w", path, err) + } + + if model.Name == "" { + continue + } + + models[model.Name] = model + } + + return models, nil +} + +// LoadModel loads a single model configuration from a file path. +func (l *Loader) LoadModel(path string) (*ModelConfig, error) { + return l.loadModel(path) +} + +func (l *Loader) loadModel(path string) (*ModelConfig, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("failed to read model file: %w", err) + } + + var model ModelConfig + if err := yaml.Unmarshal(data, &model); err != nil { + return nil, fmt.Errorf("failed to parse model config: %w", err) + } + + if model.ModelPath == "" { + return nil, fmt.Errorf("model_path is required") + } + + return &model, nil +} + +// Validate checks if the configuration is valid. +func (c *Config) Validate() error { + if c.DefaultModel == "" { + return fmt.Errorf("default_model is required") + } + + if c.Paths.LlamaServer == "" { + return fmt.Errorf("paths.llama_server is required") + } + + if c.Paths.ModelsDir == "" { + return fmt.Errorf("paths.models_dir is required") + } + + return nil +} + +// DefaultConfig returns a Config with sensible default values. +func DefaultConfig() Config { + return Config{ + Server: ServerConfig{ + ListenAddr: ":8080", + LlamaServerPort: 8081, + }, + Paths: PathsConfig{ + LlamaServer: "llama-server", + ModelsDir: "./models", + }, + DefaultModel: "", + Timeouts: TimeoutsConfig{ + ShutdownSec: 10, + StartupSec: 120, + HealthCheckIntervalMs: 1000, + }, + } +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..1f70fef --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,271 @@ +package config + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestLoadConfig(t *testing.T) { + t.Run("valid config file", func(t *testing.T) { + tmpDir := t.TempDir() + configPath := filepath.Join(tmpDir, "config.yaml") + configContent := ` +server: + listen_addr: ":8080" + llama_server_port: 8081 +paths: + llama_server: "/usr/bin/llama-server" + models_dir: "./models" +default_model: "test-model" +timeouts: + shutdown: 10 + startup: 120 + health_check_interval: 1000 +` + + err := os.WriteFile(configPath, []byte(configContent), 0644) + if err != nil { + t.Fatal(err) + } + + loader := NewLoader() + config, err := loader.LoadConfig(configPath) + + if err != nil { + t.Fatalf("LoadConfig() error = %v", err) + } + + if config.Server.ListenAddr != ":8080" { + t.Errorf("ListenAddr = %v, want :8080", config.Server.ListenAddr) + } + + if config.Server.LlamaServerPort != 8081 { + t.Errorf("LlamaServerPort = %v, want 8081", config.Server.LlamaServerPort) + } + + if config.Paths.LlamaServer != "/usr/bin/llama-server" { + t.Errorf("LlamaServer = %v, want /usr/bin/llama-server", config.Paths.LlamaServer) + } + + if config.Paths.ModelsDir != "./models" { + t.Errorf("ModelsDir = %v, want ./models", config.Paths.ModelsDir) + } + + if config.DefaultModel != "test-model" { + t.Errorf("DefaultModel = %v, want test-model", config.DefaultModel) + } + }) + + t.Run("non-existent file", func(t *testing.T) { + loader := NewLoader() + _, err := loader.LoadConfig("/nonexistent/config.yaml") + + if err == nil { + t.Fatal("expected error for non-existent file") + } + }) + + t.Run("invalid yaml", func(t *testing.T) { + tmpDir := t.TempDir() + configPath := filepath.Join(tmpDir, "config.yaml") + + err := os.WriteFile(configPath, []byte("invalid yaml content"), 0644) + if err != nil { + t.Fatal(err) + } + + loader := NewLoader() + _, err = loader.LoadConfig(configPath) + + if err == nil { + t.Fatal("expected error for invalid yaml") + } + }) +} + +func TestLoadModel(t *testing.T) { + t.Run("valid model config", func(t *testing.T) { + tmpDir := t.TempDir() + modelPath := filepath.Join(tmpDir, "model.yaml") + modelContent := ` +name: test-model +model_path: /data/model.gguf +args: + - --ctx-size + - "8192" + - --n-gpu-layers + - "99" +` + + err := os.WriteFile(modelPath, []byte(modelContent), 0644) + if err != nil { + t.Fatal(err) + } + + loader := NewLoader() + model, err := loader.LoadModel(modelPath) + + if err != nil { + t.Fatalf("LoadModel() error = %v", err) + } + + if model.Name != "test-model" { + t.Errorf("Name = %v, want test-model", model.Name) + } + + if model.ModelPath != "/data/model.gguf" { + t.Errorf("ModelPath = %v, want /data/model.gguf", model.ModelPath) + } + + expectedArgs := []string{"-m", "/data/model.gguf", "--ctx-size", "8192", "--n-gpu-layers", "99"} + actualArgs := model.BuildArgs() + if len(actualArgs) != len(expectedArgs) { + t.Errorf("Args length = %d, want %d", len(actualArgs), len(expectedArgs)) + } + for i, arg := range expectedArgs { + if actualArgs[i] != arg { + t.Errorf("Args[%d] = %v, want %v", i, actualArgs[i], arg) + } + } + }) + + t.Run("missing model_path", func(t *testing.T) { + tmpDir := t.TempDir() + modelPath := filepath.Join(tmpDir, "model.yaml") + content := "name: test-model\n" + + err := os.WriteFile(modelPath, []byte(content), 0644) + if err != nil { + t.Fatal(err) + } + + loader := NewLoader() + _, err = loader.LoadModel(modelPath) + + if err == nil { + t.Fatal("expected error for missing model_path") + } + }) +} + +func TestLoadModels(t *testing.T) { + t.Run("load multiple models", func(t *testing.T) { + tmpDir := t.TempDir() + modelsDir := filepath.Join(tmpDir, "models") + err := os.Mkdir(modelsDir, 0755) + if err != nil { + t.Fatal(err) + } + + model1 := filepath.Join(modelsDir, "model1.yaml") + model2 := filepath.Join(modelsDir, "model2.yml") + + os.WriteFile(model1, []byte("name: model1\nmodel_path: /data/model1.gguf\nargs: []\n"), 0644) + os.WriteFile(model2, []byte("name: model2\nmodel_path: /data/model2.gguf\nargs: []\n"), 0644) + + loader := NewLoader() + models, err := loader.LoadModels(modelsDir) + + if err != nil { + t.Fatalf("LoadModels() error = %v", err) + } + + if len(models) != 2 { + t.Errorf("LoadModels() returned %d models, want 2", len(models)) + } + + if models["model1"] == nil { + t.Error("model1 not found in loaded models") + } + + if models["model2"] == nil { + t.Error("model2 not found in loaded models") + } + }) + + t.Run("empty models directory", func(t *testing.T) { + tmpDir := t.TempDir() + modelsDir := filepath.Join(tmpDir, "models") + err := os.Mkdir(modelsDir, 0755) + if err != nil { + t.Fatal(err) + } + + loader := NewLoader() + models, err := loader.LoadModels(modelsDir) + + if err != nil { + t.Fatalf("LoadModels() error = %v", err) + } + + if len(models) != 0 { + t.Errorf("LoadModels() returned %d models, want 0", len(models)) + } + }) +} + +func TestTimeoutsConfig(t *testing.T) { + t.Run("duration conversions", func(t *testing.T) { + timeouts := TimeoutsConfig{ + ShutdownSec: 10, + StartupSec: 120, + HealthCheckIntervalMs: 1000, + } + + if timeouts.ShutdownDuration() != 10*time.Second { + t.Errorf("ShutdownDuration() = %v, want 10s", timeouts.ShutdownDuration()) + } + + if timeouts.StartupDuration() != 120*time.Second { + t.Errorf("StartupDuration() = %v, want 120s", timeouts.StartupDuration()) + } + + if timeouts.HealthCheckInterval() != 1*time.Second { + t.Errorf("HealthCheckInterval() = %v, want 1s", timeouts.HealthCheckInterval()) + } + }) +} + +func TestConfigValidate(t *testing.T) { + t.Run("valid config", func(t *testing.T) { + config := Config{ + DefaultModel: "test", + Paths: PathsConfig{ + LlamaServer: "/bin/llama-server", + ModelsDir: "./models", + }, + } + + if err := config.Validate(); err != nil { + t.Errorf("Validate() error = %v", err) + } + }) + + t.Run("missing default model", func(t *testing.T) { + config := Config{ + Paths: PathsConfig{ + LlamaServer: "/bin/llama-server", + ModelsDir: "./models", + }, + } + + if err := config.Validate(); err == nil { + t.Fatal("expected error for missing default model") + } + }) + + t.Run("missing llama server path", func(t *testing.T) { + config := Config{ + DefaultModel: "test", + Paths: PathsConfig{ + ModelsDir: "./models", + }, + } + + if err := config.Validate(); err == nil { + t.Fatal("expected error for missing llama server path") + } + }) +} diff --git a/internal/process/manager.go b/internal/process/manager.go new file mode 100644 index 0000000..d761439 --- /dev/null +++ b/internal/process/manager.go @@ -0,0 +1,232 @@ +package process + +import ( + "context" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "os/exec" + "sync" + "syscall" + "time" + + "github.com/fvigneault/smart-llama/internal/config" +) + +// Manager manages the llama-server subprocess lifecycle. +type Manager struct { + llamaServerPath string + llamaServerPort int + timeout config.TimeoutsConfig + logger *slog.Logger + + mu sync.RWMutex + cmd *exec.Cmd + currentModelValue string +} + +// NewManager creates a new process manager. +func NewManager(llamaServerPath string, llamaServerPort int, timeout config.TimeoutsConfig, logger *slog.Logger) *Manager { + return &Manager{ + llamaServerPath: llamaServerPath, + llamaServerPort: llamaServerPort, + timeout: timeout, + logger: logger, + } +} + +// Start launches llama-server with the given model configuration. +func (m *Manager) Start(ctx context.Context, model *config.ModelConfig) error { + m.mu.Lock() + defer m.mu.Unlock() + + if m.cmd != nil && m.cmd.Process != nil { + if m.logger != nil { + m.logger.Info("killing existing llama-server process") + } + if err := m.killProcessLocked(); err != nil { + return fmt.Errorf("failed to kill existing process: %w", err) + } + } + + args := model.BuildArgs() + args = append(args, "--port", fmt.Sprintf("%d", m.llamaServerPort)) + + if m.logger != nil { + m.logger.Info("starting llama-server", "model", model.Name, "args", args) + } + + m.cmd = exec.CommandContext(ctx, m.llamaServerPath, args...) + + m.cmd.Stdout = io.Discard + m.cmd.Stderr = io.Discard + + if err := m.cmd.Start(); err != nil { + m.currentModelValue = "" + m.cmd = nil + return fmt.Errorf("failed to start llama-server: %w", err) + } + + m.currentModelValue = model.Name + + go m.waitForExit() + + if err := m.waitForReady(ctx); err != nil { + m.currentModelValue = "" + m.killProcessLocked() + return fmt.Errorf("llama-server failed to become ready: %w", err) + } + + if m.logger != nil { + m.logger.Info("llama-server ready", "model", model.Name) + } + + return nil +} + +// Stop gracefully stops the running llama-server. +func (m *Manager) Stop() error { + m.mu.Lock() + defer m.mu.Unlock() + + if m.cmd == nil || m.cmd.Process == nil { + return nil + } + + if m.logger != nil { + m.logger.Info("stopping llama-server") + } + + return m.killProcessLocked() +} + +// CurrentModel returns the name of the currently loaded model. +func (m *Manager) CurrentModel() string { + m.mu.RLock() + defer m.mu.RUnlock() + return m.currentModelValue +} + +// IsRunning checks if llama-server is currently running. +func (m *Manager) IsRunning() bool { + m.mu.RLock() + defer m.mu.RUnlock() + + if m.cmd == nil || m.cmd.Process == nil { + return false + } + + return m.isProcessRunning(m.cmd.Process) +} + +// Health checks if llama-server is healthy. +func (m *Manager) Health() error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + url := fmt.Sprintf("http://localhost:%d/health", m.llamaServerPort) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return err + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("health check failed with status: %d", resp.StatusCode) + } + + return nil +} + +// waitForExit monitors the subprocess and updates state. +func (m *Manager) waitForExit() { + if m.cmd == nil || m.cmd.Process == nil { + return + } + + err := m.cmd.Wait() + + m.mu.Lock() + defer m.mu.Unlock() + + if err != nil { + if m.logger != nil { + m.logger.Warn("llama-server process exited", "error", err) + } + } else { + if m.logger != nil { + m.logger.Info("llama-server process exited gracefully") + } + } + + m.currentModelValue = "" +} + +// killProcessLocked terminates the llama-server process (must hold lock). +func (m *Manager) killProcessLocked() error { + if m.cmd == nil || m.cmd.Process == nil { + return nil + } + + err := m.cmd.Process.Signal(syscall.SIGTERM) + if err != nil { + if m.logger != nil { + m.logger.Warn("SIGTERM failed, sending SIGKILL", "error", err) + } + return m.cmd.Process.Kill() + } + + done := make(chan error) + go func() { + done <- m.cmd.Wait() + }() + + select { + case <-time.After(m.timeout.ShutdownDuration()): + if m.logger != nil { + m.logger.Warn("process did not exit gracefully, sending SIGKILL") + } + return m.cmd.Process.Kill() + case <-done: + return nil + } +} + +// isProcessRunning checks if a process is still alive. +func (m *Manager) isProcessRunning(p *os.Process) bool { + if p == nil { + return false + } + return p.Signal(syscall.Signal(0)) == nil +} + +// waitForReady polls the health endpoint until llama-server is ready. +func (m *Manager) waitForReady(ctx context.Context) error { + ticker := time.NewTicker(m.timeout.HealthCheckInterval()) + defer ticker.Stop() + + deadline, ok := ctx.Deadline() + if !ok { + deadline = time.Now().Add(m.timeout.StartupDuration()) + } + + for time.Now().Before(deadline) { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + if err := m.Health(); err == nil { + return nil + } + } + } + + return fmt.Errorf("timeout waiting for llama-server to be ready") +} diff --git a/internal/process/manager_test.go b/internal/process/manager_test.go new file mode 100644 index 0000000..4ecff1a --- /dev/null +++ b/internal/process/manager_test.go @@ -0,0 +1,181 @@ +package process + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "os/exec" + "syscall" + "testing" + "time" + + "github.com/fvigneault/smart-llama/internal/config" +) + +func TestNewManager(t *testing.T) { + timeouts := config.DefaultConfig().Timeouts + manager := NewManager("llama-server", 8081, timeouts, nil) + + if manager == nil { + t.Fatal("NewManager() returned nil") + } + + if !manager.IsRunning() { + if manager.CurrentModel() != "" { + t.Error("CurrentModel should be empty for stopped manager") + } + } +} + +func TestCurrentModel(t *testing.T) { + timeouts := config.DefaultConfig().Timeouts + manager := NewManager("llama-server", 8081, timeouts, nil) + + if manager.CurrentModel() != "" { + t.Errorf("CurrentModel() = %v, want empty string", manager.CurrentModel()) + } +} + +func TestIsRunning(t *testing.T) { + t.Run("no process running", func(t *testing.T) { + timeouts := config.DefaultConfig().Timeouts + manager := NewManager("llama-server", 8081, timeouts, nil) + + if manager.IsRunning() { + t.Error("IsRunning() = true, want false") + } + }) + + t.Run("mock process running", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + cmd := exec.CommandContext(ctx, "sleep", "1") + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + + cmd.Process.Kill() + }) +} + +func TestStop(t *testing.T) { + timeouts := config.DefaultConfig().Timeouts + manager := NewManager("llama-server", 8081, timeouts, nil) + + err := manager.Stop() + if err != nil { + t.Errorf("Stop() error = %v", err) + } +} + +func TestStart(t *testing.T) { + t.Run("fails with invalid executable", func(t *testing.T) { + ctx := context.Background() + timeouts := config.DefaultConfig().Timeouts + manager := NewManager("invalid-executable-123456", 8081, timeouts, nil) + + model := &config.ModelConfig{ + Name: "test", + ModelPath: "/data/model.gguf", + } + + err := manager.Start(ctx, model) + if err == nil { + t.Fatal("expected error for invalid executable") + } + + if manager.IsRunning() { + t.Error("Expected manager to remain stopped after failed start") + } + }) + + t.Run("starts process and becomes ready", func(t *testing.T) { + healthServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/health" { + w.WriteHeader(http.StatusOK) + } + })) + defer healthServer.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + timeouts := config.DefaultConfig().Timeouts + timeouts.StartupSec = 10 + timeouts.HealthCheckIntervalMs = 100 + healthPort := healthServer.Listener.Addr().(*net.TCPAddr).Port + manager := NewManager("true", healthPort, timeouts, nil) + + model := &config.ModelConfig{ + Name: "test-model", + ModelPath: "/data/model.gguf", + } + + err := manager.Start(ctx, model) + if err != nil { + t.Fatalf("Start() error = %v", err) + } + + if manager.CurrentModel() != "test-model" { + t.Errorf("CurrentModel() = %v, want test-model", manager.CurrentModel()) + } + + manager.Stop() + }) +} + +func TestHealth(t *testing.T) { + t.Run("success when server responds", func(t *testing.T) { + healthServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer healthServer.Close() + + timeouts := config.DefaultConfig().Timeouts + healthPort := healthServer.Listener.Addr().(*net.TCPAddr).Port + manager := NewManager("llama-server", healthPort, timeouts, nil) + + err := manager.Health() + if err != nil { + t.Errorf("Health() error = %v", err) + } + }) + + t.Run("fails when server not running", func(t *testing.T) { + timeouts := config.DefaultConfig().Timeouts + manager := NewManager("llama-server", 9999, timeouts, nil) + + err := manager.Health() + if err == nil { + t.Fatal("expected error when server not running") + } + }) +} + +func TestKillProcess(t *testing.T) { + t.Run("no process to kill", func(t *testing.T) { + timeouts := config.DefaultConfig().Timeouts + manager := NewManager("llama-server", 8081, timeouts, nil) + + err := manager.Stop() + if err != nil { + t.Errorf("Stop() error = %v", err) + } + }) +} + +func TestSignalZero(t *testing.T) { + cmd := exec.Command("echo", "test") + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + + err := cmd.Process.Signal(syscall.Signal(0)) + if err != nil { + t.Errorf("Signal(0) failed: %v", err) + } + + cmd.Wait() +} diff --git a/internal/proxy/handler.go b/internal/proxy/handler.go new file mode 100644 index 0000000..88d4ee4 --- /dev/null +++ b/internal/proxy/handler.go @@ -0,0 +1,33 @@ +package proxy + +import ( + "log" + "net/http" + "net/http/httputil" + "net/url" +) + +// Handler implements an HTTP reverse proxy. +type Handler struct { + target *url.URL + proxy *httputil.ReverseProxy +} + +// NewHandler creates a new reverse proxy handler. +func NewHandler(targetHost string) *Handler { + target, _ := url.Parse("http://" + targetHost) + + proxy := httputil.NewSingleHostReverseProxy(target) + + proxy.ErrorLog = log.Default() + + return &Handler{ + target: target, + proxy: proxy, + } +} + +// ServeHTTP handles the HTTP request by proxying to the target host. +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.proxy.ServeHTTP(w, r) +} diff --git a/internal/proxy/handler_test.go b/internal/proxy/handler_test.go new file mode 100644 index 0000000..bf84d31 --- /dev/null +++ b/internal/proxy/handler_test.go @@ -0,0 +1,110 @@ +package proxy + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "testing" +) + +func TestHandler(t *testing.T) { + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write([]byte("backend response")) + })) + defer backend.Close() + + backendURL, _ := url.Parse(backend.URL) + handler := NewHandler(backendURL.Host) + + tests := []struct { + name string + method string + path string + body string + expectedStatus int + expectedBody string + }{ + {"GET request", "GET", "/test", "", http.StatusOK, "backend response"}, + {"POST request", "POST", "/api/chat", "", http.StatusOK, "backend response"}, + {"PUT request", "PUT", "/update", "", http.StatusOK, "backend response"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(tt.method, tt.path, nil) + rec := httptest.NewRecorder() + + handler.ServeHTTP(rec, req) + + if rec.Code != tt.expectedStatus { + t.Errorf("status = %d, want %d", rec.Code, tt.expectedStatus) + } + + body, _ := io.ReadAll(rec.Body) + if string(body) != tt.expectedBody { + t.Errorf("body = %s, want %s", string(body), tt.expectedBody) + } + }) + } +} + +func TestHandlerHeaders(t *testing.T) { + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusBadRequest) + return + } + w.Header().Set("X-Response-Header", "value") + w.WriteHeader(http.StatusOK) + })) + defer backend.Close() + + backendURL, _ := url.Parse(backend.URL) + handler := NewHandler(backendURL.Host) + + req := httptest.NewRequest(http.MethodGet, "/test", nil) + + rec := httptest.NewRecorder() + + handler.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Errorf("status = %d, want %d", rec.Code, http.StatusOK) + } + + responseHeader := rec.Header().Get("X-Response-Header") + if responseHeader != "value" { + t.Errorf("X-Response-Header = %s, want value", responseHeader) + } +} + +func TestNewHandler(t *testing.T) { + handler := NewHandler("localhost:8081") + + if handler == nil { + t.Fatal("NewHandler() returned nil") + } + + if handler.target == nil { + t.Error("target is nil") + } + + if handler.proxy == nil { + t.Error("proxy is nil") + } +} + +func TestHandlerBackendDown(t *testing.T) { + handler := NewHandler("localhost:99999") + + req := httptest.NewRequest(http.MethodGet, "/test", nil) + rec := httptest.NewRecorder() + + handler.ServeHTTP(rec, req) + + if rec.Code == http.StatusOK { + t.Error("expected non-OK status when backend is down") + } +} diff --git a/internal/server/server.go b/internal/server/server.go new file mode 100644 index 0000000..3e32ccd --- /dev/null +++ b/internal/server/server.go @@ -0,0 +1,72 @@ +package server + +import ( + "context" + "encoding/json" + "log/slog" + "net/http" + "sync" + + "github.com/fvigneault/smart-llama/internal/config" + "github.com/fvigneault/smart-llama/internal/proxy" +) + +// Server manages the HTTP server. +type Server struct { + models map[string]*config.ModelConfig + processMgr processManager + proxyHandler *proxy.Handler + listenAddr string + logger *slog.Logger + mu sync.RWMutex +} + +type processManager interface { + Start(ctx context.Context, model *config.ModelConfig) error + Stop() error + CurrentModel() string + IsRunning() bool + Health() error +} + +// NewServer creates a new HTTP server. +func NewServer( + listenAddr string, + models map[string]*config.ModelConfig, + processMgr processManager, + proxyHandler *proxy.Handler, + logger *slog.Logger, +) *Server { + return &Server{ + models: models, + processMgr: processMgr, + proxyHandler: proxyHandler, + listenAddr: listenAddr, + logger: logger, + } +} + +// HandleModelsList returns the list of available models. +func (s *Server) HandleModelsList(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + models := make([]map[string]string, 0, len(s.models)) + for name := range s.models { + models = append(models, map[string]string{ + "id": name, + "type": "model", + }) + } + + response := map[string]interface{}{ + "object": "list", + "data": models, + } + + json.NewEncoder(w).Encode(response) +} + +// HandleHealth returns the health status. +func (s *Server) HandleHealth(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) +} diff --git a/internal/server/server_test.go b/internal/server/server_test.go new file mode 100644 index 0000000..5b0f4a9 --- /dev/null +++ b/internal/server/server_test.go @@ -0,0 +1,127 @@ +package server + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/fvigneault/smart-llama/internal/config" +) + +func TestHandleModelsList(t *testing.T) { + models := map[string]*config.ModelConfig{ + "model1": {Name: "model1", ModelPath: "/data/model1.gguf"}, + "model2": {Name: "model2", ModelPath: "/data/model2.gguf"}, + } + + server := NewServer(":8080", models, nil, nil, nil) + + req := httptest.NewRequest(http.MethodGet, "/v1/models", nil) + rec := httptest.NewRecorder() + + server.HandleModelsList(rec, req) + + if rec.Code != http.StatusOK { + t.Errorf("status = %d, want %d", rec.Code, http.StatusOK) + } + + var response struct { + Object string `json:"object"` + Data []struct { + ID string `json:"id"` + Type string `json:"type"` + } `json:"data"` + } + + if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil { + t.Fatalf("failed to parse response: %v", err) + } + + if response.Object != "list" { + t.Errorf("object = %s, want list", response.Object) + } + + if len(response.Data) != 2 { + t.Errorf("data length = %d, want 2", len(response.Data)) + } +} + +func TestHandleHealth(t *testing.T) { + server := NewServer(":8080", nil, nil, nil, nil) + + req := httptest.NewRequest(http.MethodGet, "/health", nil) + rec := httptest.NewRecorder() + + server.HandleHealth(rec, req) + + if rec.Code != http.StatusOK { + t.Errorf("status = %d, want %d", rec.Code, http.StatusOK) + } +} + +func TestHandleHealthWithProcessManager(t *testing.T) { + mockProcessManager := &mockProcessManager{running: true} + server := NewServer(":8080", nil, mockProcessManager, nil, nil) + + req := httptest.NewRequest(http.MethodGet, "/health", nil) + rec := httptest.NewRecorder() + + server.HandleHealth(rec, req) + + if rec.Code != http.StatusOK { + t.Errorf("status = %d, want %d", rec.Code, http.StatusOK) + } +} + +func TestNewServer(t *testing.T) { + models := map[string]*config.ModelConfig{"model1": {Name: "model1"}} + server := NewServer(":8080", models, nil, nil, nil) + + if server == nil { + t.Fatal("NewServer() returned nil") + } + + if server.listenAddr != ":8080" { + t.Error("listenAddr not set correctly") + } +} + +func TestHandleModelsListEmpty(t *testing.T) { + models := map[string]*config.ModelConfig{} + server := NewServer(":8080", models, nil, nil, nil) + + req := httptest.NewRequest(http.MethodGet, "/v1/models", nil) + rec := httptest.NewRecorder() + + server.HandleModelsList(rec, req) + + if rec.Code != http.StatusOK { + t.Errorf("status = %d, want %d", rec.Code, http.StatusOK) + } +} + +type mockProcessManager struct { + running bool +} + +func (m *mockProcessManager) Start(ctx context.Context, model *config.ModelConfig) error { + return nil +} + +func (m *mockProcessManager) Stop() error { + return nil +} + +func (m *mockProcessManager) CurrentModel() string { + return "" +} + +func (m *mockProcessManager) IsRunning() bool { + return m.running +} + +func (m *mockProcessManager) Health() error { + return nil +} diff --git a/models/llama3-8b.yaml b/models/llama3-8b.yaml new file mode 100644 index 0000000..28562d3 --- /dev/null +++ b/models/llama3-8b.yaml @@ -0,0 +1,15 @@ +name: llama3-8b + +model_path: /data/models/Meta-Llama-3-8B-Instruct-Q4_K_M.gguf + +args: + - --ctx-size + - "8192" + - --n-gpu-layers + - "99" + - --flash-attn + - "on" + - --batch-size + - "2048" + - --threads + - "8" \ No newline at end of file