diff --git a/src/pkg/cli/client/mock.go b/src/pkg/cli/client/mock.go index 272752155..048e1862f 100644 --- a/src/pkg/cli/client/mock.go +++ b/src/pkg/cli/client/mock.go @@ -63,6 +63,10 @@ func (MockProvider) UpdateShardDomain(ctx context.Context) error { return nil } +func (MockProvider) AccountInfo(context.Context) (*AccountInfo, error) { + return &AccountInfo{}, nil +} + func (MockProvider) GetStackName() string { return "test" } diff --git a/src/pkg/cli/compose/fixup.go b/src/pkg/cli/compose/fixup.go index 7b3e02f86..615cae441 100644 --- a/src/pkg/cli/compose/fixup.go +++ b/src/pkg/cli/compose/fixup.go @@ -37,6 +37,12 @@ func FixupServices(ctx context.Context, provider client.Provider, project *compo } slices.Sort(config.Names) // sort for binary search + accountInfo, err := provider.AccountInfo(ctx) + if err != nil { + term.Debugf("failed to get account info to fixup services: %v", err) + accountInfo = &client.AccountInfo{} + } + // Fixup any pseudo services (this might create port configs, which will affect service name replacement by ReplaceServiceNameWithDNS) for _, svccfg := range project.Services { repo := GetImageRepo(svccfg.Image) @@ -63,7 +69,7 @@ func FixupServices(ctx context.Context, provider client.Provider, project *compo } if svccfg.Provider != nil && svccfg.Provider.Type == "model" && svccfg.Image == "" && svccfg.Build == nil { - fixupModelProvider(&svccfg, project) + fixupModelProvider(&svccfg, project, accountInfo) } if _, llm := svccfg.Extensions["x-defang-llm"]; llm { @@ -87,7 +93,7 @@ func FixupServices(ctx context.Context, provider client.Provider, project *compo for name, model := range project.Models { model.Name = name // ensure the model has a name - svccfg := fixupModel(model, project) + svccfg := fixupModel(model, project, accountInfo) project.Services[svccfg.Name] = *svccfg } @@ -227,11 +233,25 @@ func parsePortString(port string) (uint32, error) { } } +const liteLLMPort uint32 = 4000 + func fixupLLM(svccfg *composeTypes.ServiceConfig) { - image := GetImageRepo(svccfg.Image) - if strings.HasSuffix(image, "/openai-access-gateway") && len(svccfg.Ports) == 0 { + // Strip tag/digest: only remove the suffix after ':' or '@' if it appears after the last '/' + // so that a registry port (e.g. registry.example:5000/litellm:latest) is handled correctly. + // Check '@' before ':' so that digest refs (name@sha256:hex) are cut at the '@', not inside the hex. + sanitizedImage := svccfg.Image + lastSlash := strings.LastIndex(sanitizedImage, "/") + suffix := sanitizedImage[lastSlash+1:] + if i := strings.IndexByte(suffix, '@'); i >= 0 { + sanitizedImage = sanitizedImage[:lastSlash+1+i] + } else if i := strings.IndexByte(suffix, ':'); i >= 0 { + sanitizedImage = sanitizedImage[:lastSlash+1+i] + } + sanitizedImage = strings.ToLower(sanitizedImage) + if strings.HasSuffix(sanitizedImage, "/litellm") && len(svccfg.Ports) == 0 { // HACK: we must have at least one host port to get a CNAME for the service - var port uint32 = 80 + // litellm listens on 4000 by default + var port uint32 = liteLLMPort term.Debugf("service %q: adding LLM host port %d", svccfg.Name, port) svccfg.Ports = []composeTypes.ServicePortConfig{{Target: port, Mode: Mode_HOST, Protocol: Protocol_TCP}} } @@ -339,40 +359,75 @@ func fixupIngressPorts(svccfg *composeTypes.ServiceConfig) { // Declare a private network for the model provider const modelProviderNetwork = "model_provider_private" -func fixupModel(model composeTypes.ModelConfig, project *composeTypes.Project) *composeTypes.ServiceConfig { +func fixupModel(model composeTypes.ModelConfig, project *composeTypes.Project, info *client.AccountInfo) *composeTypes.ServiceConfig { svccfg := &composeTypes.ServiceConfig{ Name: model.Name, Extensions: model.Extensions, } - makeAccessGatewayService(svccfg, project, model.Model) // TODO: pass other model options too + makeAccessGatewayService(svccfg, project, model.Model, info) // TODO: pass other model options too return svccfg } -func fixupModelProvider(svccfg *composeTypes.ServiceConfig, project *composeTypes.Project) { +func fixupModelProvider(svccfg *composeTypes.ServiceConfig, project *composeTypes.Project, info *client.AccountInfo) { var model string if modelVals := svccfg.Provider.Options["model"]; len(modelVals) == 1 { model = modelVals[0] } - makeAccessGatewayService(svccfg, project, model) + makeAccessGatewayService(svccfg, project, model, info) } -func makeAccessGatewayService(svccfg *composeTypes.ServiceConfig, project *composeTypes.Project, model string) { +func makeAccessGatewayService(svccfg *composeTypes.ServiceConfig, project *composeTypes.Project, model string, info *client.AccountInfo) { // Local Docker sets [SERVICE]_URL and [SERVICE]_MODEL environment variables on the dependent services envName := strings.ToUpper(svccfg.Name) // TODO: handle characters that are not allowed in env vars, like '-' endpointEnvVar := envName + "_URL" - urlVal := "http://" + svccfg.Name + "/api/v1/" + urlVal := "http://" + svccfg.Name + ":" + strconv.FormatUint(uint64(liteLLMPort), 10) + "/v1/" modelEnvVar := envName + "_MODEL" - empty := "" + resolvedModel, masterKey := configureAccessGateway(svccfg, project, model, info) + wireDependentServices(project, svccfg.Name, urlVal, resolvedModel, masterKey, endpointEnvVar, modelEnvVar) +} + +// configureAccessGateway resolves the model name for the target provider, configures +// the LiteLLM container (image, command, network, port), and derives LITELLM_MASTER_KEY. +// Returns the resolved model string and the master key pointer. +func configureAccessGateway(svccfg *composeTypes.ServiceConfig, project *composeTypes.Project, model string, info *client.AccountInfo) (string, *string) { // svccfg.Deploy.Resources.Reservations.Limits = &composeTypes.Resources{} TODO: avoid memory limits warning if svccfg.Environment == nil { svccfg.Environment = composeTypes.MappingWithEquals{} } - if _, exists := svccfg.Environment["OPENAI_API_KEY"]; !exists { - svccfg.Environment["OPENAI_API_KEY"] = &empty // disable auth; see https://github.com/DefangLabs/openai-access-gateway/pull/5 + + alias := model + switch info.Provider { + case client.ProviderAWS: + switch model { + case "chat-default": + model = "us.amazon.nova-2-lite-v1:0" + case "embedding-default": + model = "amazon.titan-embed-text-v2:0" + } + model = modelWithProvider(model, "bedrock") + if info.Region != "" { + svccfg.Environment["AWS_REGION"] = &info.Region + } + case client.ProviderGCP: + switch model { + case "chat-default": + model = "gemini-2.5-flash" + case "embedding-default": + model = "gemini-embedding-001" + } + model = modelWithProvider(model, "vertex_ai") + if info.AccountID != "" { + svccfg.Environment["VERTEXAI_PROJECT"] = &info.AccountID + } + if info.Region != "" { + svccfg.Environment["VERTEXAI_LOCATION"] = &info.Region + } } + // svccfg.HealthCheck = &composeTypes.ServiceHealthCheckConfig{} TODO: add healthcheck - svccfg.Image = "defangio/openai-access-gateway" + svccfg.Image = "litellm/litellm:v1.82.3-stable.patch.3" + svccfg.Command = []string{"--drop_params", "--model", model, "--alias", alias} if svccfg.Networks == nil { // New compose-go versions do not create networks for "provider:" services, so we need to create it here svccfg.Networks = make(map[string]*composeTypes.ServiceNetworkConfig) @@ -380,13 +435,44 @@ func makeAccessGatewayService(svccfg *composeTypes.ServiceConfig, project *compo delete(svccfg.Networks, "default") // remove the default network } svccfg.Networks[modelProviderNetwork] = nil - svccfg.Ports = []composeTypes.ServicePortConfig{{Target: 80, Mode: Mode_HOST, Protocol: Protocol_TCP}} + svccfg.Ports = []composeTypes.ServicePortConfig{{Target: liteLLMPort, Mode: Mode_HOST, Protocol: Protocol_TCP}} svccfg.Provider = nil // remove "provider:" because current backend will not accept it project.Networks[modelProviderNetwork] = composeTypes.NetworkConfig{Name: modelProviderNetwork} - // Set environment variables (url and model) for any service that depends on the model - for _, dependency := range project.Services { - if _, ok := dependency.DependsOn[svccfg.Name]; ok { + masterKey, exists := svccfg.Environment["LITELLM_MASTER_KEY"] + if !exists { + openAIKey := "" + for _, service := range project.Services { + if _, ok := service.DependsOn[svccfg.Name]; ok { + if key, ok := service.Environment["OPENAI_API_KEY"]; ok { + if openAIKey == "" { + openAIKey = *key + } else if *key != openAIKey { + term.Errorf("multiple different OPENAI_API_KEY values found in services depending on %q", svccfg.Name) + break + } + } + } + } + if openAIKey == "" { + key := "networkisalreadyprivate" + masterKey = &key + } else { + masterKey = &openAIKey + } + svccfg.Environment["LITELLM_MASTER_KEY"] = masterKey + } + + return model, masterKey +} + +// wireDependentServices injects URL, model, and API-key env vars and adds the +// model-provider network to every service that depends on svcName. +func wireDependentServices(project *composeTypes.Project, svcName, urlVal, model string, masterKey *string, endpointEnvVar, modelEnvVar string) { + for name, dependency := range project.Services { + changed := false + + if _, ok := dependency.DependsOn[svcName]; ok { if dependency.Environment == nil { dependency.Environment = make(composeTypes.MappingWithEquals) } @@ -397,9 +483,13 @@ func makeAccessGatewayService(svccfg *composeTypes.ServiceConfig, project *compo if _, ok := dependency.Environment[modelEnvVar]; !ok && model != "" { dependency.Environment[modelEnvVar] = &model } + if _, ok := dependency.Environment["OPENAI_API_KEY"]; !ok { + dependency.Environment["OPENAI_API_KEY"] = masterKey + } + changed = true } - if modelDep, ok := dependency.Models[svccfg.Name]; ok { + if modelDep, ok := dependency.Models[svcName]; ok { endpointVar := endpointEnvVar if modelDep != nil && modelDep.EndpointVariable != "" { endpointVar = modelDep.EndpointVariable @@ -419,19 +509,31 @@ func makeAccessGatewayService(svccfg *composeTypes.ServiceConfig, project *compo dependency.Environment[modelVar] = &model } // If the model is not already declared as a dependency, add it - if _, ok := dependency.DependsOn[svccfg.Name]; !ok { + if _, ok := dependency.DependsOn[svcName]; !ok { if dependency.DependsOn == nil { dependency.DependsOn = make(map[string]composeTypes.ServiceDependency) } - dependency.DependsOn[svccfg.Name] = composeTypes.ServiceDependency{ + dependency.DependsOn[svcName] = composeTypes.ServiceDependency{ Condition: composeTypes.ServiceConditionStarted, Required: true, } } + changed = true + } + + if changed { + project.Services[name] = dependency } } } +func modelWithProvider(model, prefix string) string { + if strings.Contains(model, "/") { + return model // already has a provider prefix + } + return prefix + "/" + model +} + func GetImageRepo(image string) string { repo, _, _ := strings.Cut(image, ":") return strings.ToLower(repo) diff --git a/src/pkg/cli/compose/fixup_test.go b/src/pkg/cli/compose/fixup_test.go index 89ca089f2..052a6a0b2 100644 --- a/src/pkg/cli/compose/fixup_test.go +++ b/src/pkg/cli/compose/fixup_test.go @@ -9,6 +9,7 @@ import ( "github.com/DefangLabs/defang/src/pkg/cli/client" composeTypes "github.com/compose-spec/compose-go/v2/types" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestFixup(t *testing.T) { @@ -43,3 +44,196 @@ func TestFixup(t *testing.T) { } }) } + +func newLLMService() composeTypes.ServiceConfig { + return composeTypes.ServiceConfig{ + Name: "llm", + Environment: composeTypes.MappingWithEquals{}, + Networks: map[string]*composeTypes.ServiceNetworkConfig{}, + } +} + +func TestMakeAccessGatewayServiceAWS(t *testing.T) { + info := &client.AccountInfo{ + Provider: client.ProviderAWS, + Region: "us-east-1", + AccountID: "123456789", + } + + t.Run("chat-default model", func(t *testing.T) { + proj := &composeTypes.Project{Networks: map[string]composeTypes.NetworkConfig{}, Services: composeTypes.Services{}} + svccfg := newLLMService() + makeAccessGatewayService(&svccfg, proj, "chat-default", info) + + require.Equal(t, []string{"--drop_params", "--model", "bedrock/us.amazon.nova-2-lite-v1:0", "--alias", "chat-default"}, []string(svccfg.Command)) + assert.Equal(t, "us-east-1", *svccfg.Environment["AWS_REGION"]) + }) + + t.Run("embedding-default model", func(t *testing.T) { + proj := &composeTypes.Project{Networks: map[string]composeTypes.NetworkConfig{}, Services: composeTypes.Services{}} + svccfg := newLLMService() + makeAccessGatewayService(&svccfg, proj, "embedding-default", info) + + require.Equal(t, []string{"--drop_params", "--model", "bedrock/amazon.titan-embed-text-v2:0", "--alias", "embedding-default"}, []string(svccfg.Command)) + }) + + t.Run("custom model gets bedrock prefix", func(t *testing.T) { + proj := &composeTypes.Project{Networks: map[string]composeTypes.NetworkConfig{}, Services: composeTypes.Services{}} + svccfg := newLLMService() + makeAccessGatewayService(&svccfg, proj, "anthropic.claude-3-5-sonnet", info) + + require.Equal(t, []string{"--drop_params", "--model", "bedrock/anthropic.claude-3-5-sonnet", "--alias", "anthropic.claude-3-5-sonnet"}, []string(svccfg.Command)) + }) + + t.Run("model with existing provider prefix is not double-prefixed", func(t *testing.T) { + proj := &composeTypes.Project{Networks: map[string]composeTypes.NetworkConfig{}, Services: composeTypes.Services{}} + svccfg := newLLMService() + makeAccessGatewayService(&svccfg, proj, "bedrock/anthropic.claude-3-5-sonnet", info) + + require.Equal(t, []string{"--drop_params", "--model", "bedrock/anthropic.claude-3-5-sonnet", "--alias", "bedrock/anthropic.claude-3-5-sonnet"}, []string(svccfg.Command)) + }) +} + +func TestMakeAccessGatewayServiceGCP(t *testing.T) { + info := &client.AccountInfo{ + Provider: client.ProviderGCP, + Region: "us-central1", + AccountID: "my-gcp-project", + } + + t.Run("chat-default model", func(t *testing.T) { + proj := &composeTypes.Project{Networks: map[string]composeTypes.NetworkConfig{}, Services: composeTypes.Services{}} + svccfg := newLLMService() + makeAccessGatewayService(&svccfg, proj, "chat-default", info) + + require.Equal(t, []string{"--drop_params", "--model", "vertex_ai/gemini-2.5-flash", "--alias", "chat-default"}, []string(svccfg.Command)) + assert.Equal(t, "my-gcp-project", *svccfg.Environment["VERTEXAI_PROJECT"]) + assert.Equal(t, "us-central1", *svccfg.Environment["VERTEXAI_LOCATION"]) + }) + + t.Run("embedding-default model", func(t *testing.T) { + proj := &composeTypes.Project{Networks: map[string]composeTypes.NetworkConfig{}, Services: composeTypes.Services{}} + svccfg := newLLMService() + makeAccessGatewayService(&svccfg, proj, "embedding-default", info) + + require.Equal(t, []string{"--drop_params", "--model", "vertex_ai/gemini-embedding-001", "--alias", "embedding-default"}, []string(svccfg.Command)) + }) +} + +func TestMakeAccessGatewayServiceLiteLLMMasterKey(t *testing.T) { + info := &client.AccountInfo{} + + t.Run("no OPENAI_API_KEY uses default key", func(t *testing.T) { + proj := &composeTypes.Project{Networks: map[string]composeTypes.NetworkConfig{}, Services: composeTypes.Services{}} + svccfg := newLLMService() + makeAccessGatewayService(&svccfg, proj, "ai/model", info) + + assert.Equal(t, "networkisalreadyprivate", *svccfg.Environment["LITELLM_MASTER_KEY"]) + }) + + t.Run("OPENAI_API_KEY on dependent service propagates to LITELLM_MASTER_KEY", func(t *testing.T) { + apiKey := "sk-my-secret-key" //nolint:gosec + proj := &composeTypes.Project{ + Networks: map[string]composeTypes.NetworkConfig{}, + Services: composeTypes.Services{ + "app": { + Name: "app", + Image: "myapp", + DependsOn: map[string]composeTypes.ServiceDependency{ + "llm": {Condition: composeTypes.ServiceConditionStarted, Required: true}, + }, + Environment: composeTypes.MappingWithEquals{"OPENAI_API_KEY": &apiKey}, + Networks: map[string]*composeTypes.ServiceNetworkConfig{}, + }, + }, + } + svccfg := newLLMService() + makeAccessGatewayService(&svccfg, proj, "ai/model", info) + + assert.Equal(t, "sk-my-secret-key", *svccfg.Environment["LITELLM_MASTER_KEY"]) + }) + + t.Run("existing LITELLM_MASTER_KEY on service is preserved", func(t *testing.T) { + proj := &composeTypes.Project{Networks: map[string]composeTypes.NetworkConfig{}, Services: composeTypes.Services{}} + existingKey := "existing-master-key" + svccfg := newLLMService() + svccfg.Environment["LITELLM_MASTER_KEY"] = &existingKey + makeAccessGatewayService(&svccfg, proj, "ai/model", info) + + assert.Equal(t, "existing-master-key", *svccfg.Environment["LITELLM_MASTER_KEY"]) + }) +} + +func TestFixupLLM(t *testing.T) { + tests := []struct { + name string + image string + existingPorts []composeTypes.ServicePortConfig + wantPort bool + }{ + { + name: "registry with port and tag adds litellm port", + image: "registry.example:5000/litellm:latest", + wantPort: true, + }, + { + name: "registry with port and no tag adds litellm port", + image: "registry.example:5000/litellm", + wantPort: true, + }, + { + name: "standard registry with path and tag adds litellm port", + image: "ghcr.io/berriai/litellm:main-latest", + wantPort: true, + }, + { + name: "image with digest adds litellm port", + image: "ghcr.io/berriai/litellm@sha256:abc123", + wantPort: true, + }, + { + name: "non-litellm image does not add port", + image: "registry.example:5000/other:tag", + wantPort: false, + }, + { + name: "litellm image with existing ports does not add port", + image: "ghcr.io/berriai/litellm:main-latest", + existingPorts: []composeTypes.ServicePortConfig{ + {Target: 8080, Mode: Mode_HOST, Protocol: Protocol_TCP}, + }, + wantPort: false, + }, + { + name: "bare image name without slash does not match", + image: "litellm:latest", + wantPort: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + svc := composeTypes.ServiceConfig{ + Name: "llm", + Image: tt.image, + Ports: tt.existingPorts, + } + fixupLLM(&svc) + if tt.wantPort { + require.Len(t, svc.Ports, 1) + assert.Equal(t, liteLLMPort, svc.Ports[0].Target) + assert.Equal(t, Mode_HOST, svc.Ports[0].Mode) + assert.Equal(t, Protocol_TCP, svc.Ports[0].Protocol) + } else { + assert.Equal(t, tt.existingPorts, svc.Ports) + } + }) + } +} + +func TestModelWithProvider(t *testing.T) { + assert.Equal(t, "bedrock/my-model", modelWithProvider("my-model", "bedrock")) + assert.Equal(t, "bedrock/my-model", modelWithProvider("bedrock/my-model", "bedrock")) + assert.Equal(t, "vertex_ai/gemini-2.5-flash", modelWithProvider("gemini-2.5-flash", "vertex_ai")) + assert.Equal(t, "vertex_ai/gemini-2.5-flash", modelWithProvider("vertex_ai/gemini-2.5-flash", "vertex_ai")) +} diff --git a/src/pkg/cli/subscribe.go b/src/pkg/cli/subscribe.go index 223965452..6b0b95d4a 100644 --- a/src/pkg/cli/subscribe.go +++ b/src/pkg/cli/subscribe.go @@ -82,13 +82,13 @@ func WaitServiceState( } } - term.Infof("Waiting for %q to be in state %s...\n", pendingServices, targetState) // TODO: don't print in Go-routine + term.Infof("Waiting for services to finish deploying: %q\n", pendingServices) // TODO: don't print in Go-routine if msg == nil { continue } - term.Debugf("service %s with state ( %s ) and status: %s\n", msg.Name, msg.State, msg.Status) // TODO: don't print in Go-routine + term.Debugf("Service update: %s: state=%s and status=%s\n", msg.Name, msg.State, msg.Status) // TODO: don't print in Go-routine if _, ok := serviceStates[msg.Name]; !ok { term.Debugf("unexpected service %s update", msg.Name) // TODO: don't print in Go-routine diff --git a/src/testdata/llm/compose.yaml b/src/testdata/llm/compose.yaml index f04919146..2dd80073a 100644 --- a/src/testdata/llm/compose.yaml +++ b/src/testdata/llm/compose.yaml @@ -1,18 +1,23 @@ services: + alt-repo: + x-defang-llm: true + image: "altrepo.com/litellm:latest" + networks: + default: null + ports: + - mode: host + target: "4000" + protocol: tcp llm: x-defang-llm: true image: "llm:latest" gateway-with-ports: x-defang-llm: true - image: "defang.io/openai-access-gateway:latest" + image: "litellm/litellm:latest" ports: - 5678:5678 gateway-without-ports: x-defang-llm: true - image: "defang.io/openai-access-gateway:latest" - - alt-repo: - x-defang-llm: true - image: "altrepo.com/openai-access-gateway:latest" + image: "litellm/litellm:latest" diff --git a/src/testdata/llm/compose.yaml.fixup b/src/testdata/llm/compose.yaml.fixup index ccc27e420..0978b0653 100644 --- a/src/testdata/llm/compose.yaml.fixup +++ b/src/testdata/llm/compose.yaml.fixup @@ -2,14 +2,14 @@ "alt-repo": { "command": null, "entrypoint": null, - "image": "altrepo.com/openai-access-gateway:latest", + "image": "altrepo.com/litellm:latest", "networks": { "default": null }, "ports": [ { "mode": "host", - "target": 80, + "target": 4000, "protocol": "tcp" } ] @@ -17,7 +17,7 @@ "gateway-with-ports": { "command": null, "entrypoint": null, - "image": "defang.io/openai-access-gateway:latest", + "image": "litellm/litellm:latest", "networks": { "default": null }, @@ -34,14 +34,14 @@ "gateway-without-ports": { "command": null, "entrypoint": null, - "image": "defang.io/openai-access-gateway:latest", + "image": "litellm/litellm:latest", "networks": { "default": null }, "ports": [ { "mode": "host", - "target": 80, + "target": 4000, "protocol": "tcp" } ] diff --git a/src/testdata/llm/compose.yaml.golden b/src/testdata/llm/compose.yaml.golden index 465ff9246..d6f462384 100644 --- a/src/testdata/llm/compose.yaml.golden +++ b/src/testdata/llm/compose.yaml.golden @@ -1,12 +1,16 @@ name: llm services: alt-repo: - image: altrepo.com/openai-access-gateway:latest + image: altrepo.com/litellm:latest networks: default: null + ports: + - mode: host + target: 4000 + protocol: tcp x-defang-llm: true gateway-with-ports: - image: defang.io/openai-access-gateway:latest + image: litellm/litellm:latest networks: default: null ports: @@ -16,7 +20,7 @@ services: protocol: tcp x-defang-llm: true gateway-without-ports: - image: defang.io/openai-access-gateway:latest + image: litellm/litellm:latest networks: default: null x-defang-llm: true diff --git a/src/testdata/models/compose.yaml.fixup b/src/testdata/models/compose.yaml.fixup index 02574ebe9..151cb4df0 100644 --- a/src/testdata/models/compose.yaml.fixup +++ b/src/testdata/models/compose.yaml.fixup @@ -1,28 +1,40 @@ { "ai_model": { - "command": null, + "command": [ + "--drop_params", + "--model", + "ai/model", + "--alias", + "ai/model" + ], "entrypoint": null, "environment": { - "OPENAI_API_KEY": "" + "LITELLM_MASTER_KEY": "networkisalreadyprivate" }, - "image": "defangio/openai-access-gateway", + "image": "litellm/litellm:v1.82.3-stable.patch.3", "networks": { "model_provider_private": null }, "ports": [ { "mode": "host", - "target": 80, + "target": 4000, "protocol": "tcp" } ] }, "modellist": { "command": null, + "depends_on": { + "ai_model": { + "condition": "service_started", + "required": true + } + }, "entrypoint": null, "environment": { "AI_MODEL_MODEL": "ai/model", - "AI_MODEL_URL": "http://mock-ai-model/api/v1/" + "AI_MODEL_URL": "http://mock-ai-model:4000/v1/" }, "image": "app", "models": { @@ -35,10 +47,16 @@ }, "modelmap": { "command": null, + "depends_on": { + "ai_model": { + "condition": "service_started", + "required": true + } + }, "entrypoint": null, "environment": { "AI_MODEL_MODEL": "ai/model", - "AI_MODEL_URL": "http://mock-ai-model/api/v1/" + "AI_MODEL_URL": "http://mock-ai-model:4000/v1/" }, "image": "app", "models": { @@ -50,28 +68,40 @@ } }, "my_model": { - "command": null, + "command": [ + "--drop_params", + "--model", + "ai/model", + "--alias", + "ai/model" + ], "entrypoint": null, "environment": { - "OPENAI_API_KEY": "" + "LITELLM_MASTER_KEY": "networkisalreadyprivate" }, - "image": "defangio/openai-access-gateway", + "image": "litellm/litellm:v1.82.3-stable.patch.3", "networks": { "model_provider_private": null }, "ports": [ { "mode": "host", - "target": 80, + "target": 4000, "protocol": "tcp" } ] }, "withendpoint": { "command": null, + "depends_on": { + "my_model": { + "condition": "service_started", + "required": true + } + }, "entrypoint": null, "environment": { - "MODEL_URL": "http://mock-my-model/api/v1/", + "MODEL_URL": "http://mock-my-model:4000/v1/", "MY_MODEL_MODEL": "ai/model" }, "image": "app", diff --git a/src/testdata/models/compose.yaml.warnings b/src/testdata/models/compose.yaml.warnings index c514b394f..83f7405d9 100644 --- a/src/testdata/models/compose.yaml.warnings +++ b/src/testdata/models/compose.yaml.warnings @@ -1,5 +1,7 @@ + ! service "ai_model": environment "LITELLM_MASTER_KEY" may contain sensitive information; consider using 'defang config set LITELLM_MASTER_KEY' to securely store this value ! service "ai_model": missing memory reservation; using provider-specific defaults. Specify deploy.resources.reservations.memory to avoid out-of-memory errors ! service "modellist": missing memory reservation; using provider-specific defaults. Specify deploy.resources.reservations.memory to avoid out-of-memory errors ! service "modelmap": missing memory reservation; using provider-specific defaults. Specify deploy.resources.reservations.memory to avoid out-of-memory errors + ! service "my_model": environment "LITELLM_MASTER_KEY" may contain sensitive information; consider using 'defang config set LITELLM_MASTER_KEY' to securely store this value ! service "my_model": missing memory reservation; using provider-specific defaults. Specify deploy.resources.reservations.memory to avoid out-of-memory errors ! service "withendpoint": missing memory reservation; using provider-specific defaults. Specify deploy.resources.reservations.memory to avoid out-of-memory errors diff --git a/src/testdata/provider/compose.yaml.fixup b/src/testdata/provider/compose.yaml.fixup index d986feaf2..63cdc7081 100644 --- a/src/testdata/provider/compose.yaml.fixup +++ b/src/testdata/provider/compose.yaml.fixup @@ -1,19 +1,25 @@ { "ai_runner": { - "command": null, + "command": [ + "--drop_params", + "--model", + "ai/smollm2", + "--alias", + "ai/smollm2" + ], "entrypoint": null, "environment": { "DEBUG": "true", - "OPENAI_API_KEY": "" + "LITELLM_MASTER_KEY": "networkisalreadyprivate" }, - "image": "defangio/openai-access-gateway", + "image": "litellm/litellm:v1.82.3-stable.patch.3", "networks": { "model_provider_private": null }, "ports": [ { "mode": "host", - "target": 80, + "target": 4000, "protocol": "tcp" } ] @@ -29,7 +35,8 @@ "entrypoint": null, "environment": { "AI_RUNNER_MODEL": "ai/smollm2", - "AI_RUNNER_URL": "http://mock-ai-runner/api/v1/" + "AI_RUNNER_URL": "http://mock-ai-runner:4000/v1/", + "OPENAI_API_KEY": "networkisalreadyprivate" }, "image": "my-chat-app", "networks": { diff --git a/src/testdata/provider/compose.yaml.warnings b/src/testdata/provider/compose.yaml.warnings index 3a7067aa2..95bc4b9ba 100644 --- a/src/testdata/provider/compose.yaml.warnings +++ b/src/testdata/provider/compose.yaml.warnings @@ -1,2 +1,4 @@ + ! service "ai_runner": environment "LITELLM_MASTER_KEY" may contain sensitive information; consider using 'defang config set LITELLM_MASTER_KEY' to securely store this value ! service "ai_runner": missing memory reservation; using provider-specific defaults. Specify deploy.resources.reservations.memory to avoid out-of-memory errors + ! service "chat": environment "OPENAI_API_KEY" may contain sensitive information; consider using 'defang config set OPENAI_API_KEY' to securely store this value ! service "chat": missing memory reservation; using provider-specific defaults. Specify deploy.resources.reservations.memory to avoid out-of-memory errors