Merge pull request #75 from kerthcet/refactor/readme

Prepare for v0.0.2
InftyAI · Aug 8, 2024 · 67f78a7 · 67f78a7
2 parents 6407c4c + a2374e1
commit 67f78a7
Show file tree

Hide file tree

Showing 11 changed files with 25 additions and 31 deletions.
diff --git a/Makefile b/Makefile
@@ -178,25 +178,21 @@ image-build:
 		--build-arg BASE_IMAGE=$(BASE_IMAGE) \
 		--build-arg BUILDER_IMAGE=$(BUILDER_IMAGE) \
 		--build-arg CGO_ENABLED=$(CGO_ENABLED) \
-		$(PUSH) \
-		$(LOAD) \
 		$(IMAGE_BUILD_EXTRA_OPTS) ./
-image-push: PUSH=--push
+image-push: IMAGE_BUILD_EXTRA_OPTS=--push
 image-push: image-build
-image-load: LOAD=--load
+image-load: IMAGE_BUILD_EXTRA_OPTS=--load
 image-load: image-load
 
 .PHONY: loader-image-build
 loader-image-build:
 	$(IMAGE_BUILD_CMD) -t $(LOADER_IMG) \
 		-f Dockerfile.loader \
-		$(PUSH) \
-		$(LOAD) \
 		$(IMAGE_BUILD_EXTRA_OPTS) ./
-loader-image-push: PUSH=--push
+loader-image-push: IMAGE_BUILD_EXTRA_OPTS=--push
 loader-image-push: loader-image-build
 
-loader-image-load: LOAD=--load
+loader-image-load: IMAGE_BUILD_EXTRA_OPTS=--load
 loader-image-load: loader-image-build
 
 KIND = $(shell pwd)/bin/kind
@@ -206,7 +202,7 @@ kind:
 
 .PHONY: kind-image-build
 kind-image-build: PLATFORMS=linux/amd64
-kind-image-build: LOAD=--load
+kind-image-build: IMAGE_BUILD_EXTRA_OPTS=--load
 kind-image-build: kind image-build
 
 ##@ Deployment

diff --git a/README.md b/README.md
@@ -16,7 +16,7 @@
 ## Feature Overview
 
 - **User Friendly**: People can quick deploy a LLM service with minimal configurations.
-- **High performance**: llmaz supports a wide range of advanced inference backends for high performance, like [vLLM](https://github.com/vllm-project/vllm), [SGLang](https://github.com/sgl-project/sglang). Find the full list of supported backends [here](./docs/support-backends.md).
+- **High Performance**: llmaz supports a wide range of advanced inference backends for high performance, like [vLLM](https://github.com/vllm-project/vllm), [SGLang](https://github.com/sgl-project/sglang). Find the full list of supported backends [here](./docs/support-backends.md).
 - **Scaling Efficiency (WIP)**: llmaz works smoothly with autoscaling components like [Cluster-Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) or [Karpenter](https://github.com/kubernetes-sigs/karpenter) to support elastic scenarios.
 - **Accelerator Fungibility (WIP)**: llmaz supports serving the same LLM with various accelerators to optimize cost and performance.
 - **SOTA Inference (WIP)**: llmaz supports the latest cutting-edge researches like [Speculative Decoding](https://arxiv.org/abs/2211.17192) or [Splitwise](https://arxiv.org/abs/2311.18677) to run on Kubernetes.
@@ -35,7 +35,7 @@ is to apply the `Model` and `Playground` yamls.
 
 Please refer to **[examples](/docs/examples/README.md)** to learn more.
 
-> Note: if your model needs Huggingface token for weight downloads, please run `kubectl create secret generic model-secret  --from-literal=HF_TOKEN=<your token>` ahead.
+> Note: if your model needs Huggingface token for weight downloads, please run `kubectl create secret generic model-secret --from-literal=HF_TOKEN=<your token>` ahead.
 
 #### Model
 

diff --git a/docs/examples/vllm-huggingface/model.yaml b/docs/examples/vllm-huggingface/model.yaml
@@ -7,7 +7,6 @@ spec:
   source:
     modelHub:
       modelID: facebook/opt-125m
-      revision: main
   inferenceFlavors:
   - name: t4 # GPU type
     requests:

diff --git a/docs/installation.md b/docs/installation.md
@@ -14,7 +14,7 @@ LWS_VERSION=v0.3.0
 kubectl apply --server-side -f https://github.com/kubernetes-sigs/lws/releases/download/$LWS_VERSION/manifests.yaml
 
 # llmaz runs in llmaz-system
-LLMAZ_VERSION=v0.0.1
+LLMAZ_VERSION=v0.0.2
 kubectl apply --server-side -f https://github.com/inftyai/llmaz/releases/download/$LLMAZ_VERSION/manifests.yaml
 ```
 
@@ -24,7 +24,7 @@ kubectl apply --server-side -f https://github.com/inftyai/llmaz/releases/downloa
 LWS_VERSION=v0.3.0
 kubectl delete -f https://github.com/kubernetes-sigs/lws/releases/download/$LWS_VERSION/manifests.yaml
 
-LLMAZ_VERSION=v0.0.1
+LLMAZ_VERSION=v0.0.2
 kubectl delete -f https://github.com/inftyai/llmaz/releases/download/$LLMAZ_VERSION/manifests.yaml
 ```
 

diff --git a/pkg/controller/inference/service_controller.go b/pkg/controller/inference/service_controller.go
@@ -142,15 +142,15 @@ func buildWorkloadApplyConfiguration(service *inferenceapi.Service, model *corea
 }
 
 func injectModelProperties(template *applyconfigurationv1.LeaderWorkerTemplateApplyConfiguration, model *coreapi.Model) {
-	modelSource := modelSource.NewDataSourceProvider(model)
+	source := modelSource.NewModelSourceProvider(model)
 
 	template.WorkerTemplate.Labels = util.MergeKVs(template.WorkerTemplate.Labels, modelLabels(model))
 
-	injectModelLoader(template, modelSource)
+	injectModelLoader(template, source)
 	injectModelFlavor(template, model)
 }
 
-func injectModelLoader(template *applyconfigurationv1.LeaderWorkerTemplateApplyConfiguration, source modelSource.DataSourceProvider) {
+func injectModelLoader(template *applyconfigurationv1.LeaderWorkerTemplateApplyConfiguration, source modelSource.ModelSourceProvider) {
 	source.InjectModelLoader(template.WorkerTemplate)
 }
 

diff --git a/pkg/controller_helper/backend/sglang.go b/pkg/controller_helper/backend/sglang.go
@@ -25,7 +25,7 @@ import (
 	coreapi "inftyai.com/llmaz/api/core/v1alpha1"
 	inferenceapi "inftyai.com/llmaz/api/inference/v1alpha1"
 	"inftyai.com/llmaz/pkg"
-	source "inftyai.com/llmaz/pkg/controller_helper/model_source"
+	modelSource "inftyai.com/llmaz/pkg/controller_helper/model_source"
 )
 
 var _ Backend = (*SGLANG)(nil)
@@ -66,10 +66,10 @@ func (s *SGLANG) DefaultCommands() []string {
 }
 
 func (s *SGLANG) DefaultArgs(model *coreapi.Model) []string {
-	modelSource := source.NewDataSourceProvider(model)
+	source := modelSource.NewModelSourceProvider(model)
 	return []string{
-		"--model-path", modelSource.ModelPath(),
-		"--served-model-name", modelSource.ModelName(),
+		"--model-path", source.ModelPath(),
+		"--served-model-name", source.ModelName(),
 		"--host", "0.0.0.0",
 		"--port", strconv.Itoa(pkg.DEFAULT_BACKEND_PORT),
 	}

diff --git a/pkg/controller_helper/backend/vllm.go b/pkg/controller_helper/backend/vllm.go
@@ -25,7 +25,7 @@ import (
 	coreapi "inftyai.com/llmaz/api/core/v1alpha1"
 	inferenceapi "inftyai.com/llmaz/api/inference/v1alpha1"
 	"inftyai.com/llmaz/pkg"
-	source "inftyai.com/llmaz/pkg/controller_helper/model_source"
+	modelSource "inftyai.com/llmaz/pkg/controller_helper/model_source"
 )
 
 var _ Backend = (*VLLM)(nil)
@@ -66,10 +66,10 @@ func (v *VLLM) DefaultCommands() []string {
 }
 
 func (v *VLLM) DefaultArgs(model *coreapi.Model) []string {
-	modelSource := source.NewDataSourceProvider(model)
+	source := modelSource.NewModelSourceProvider(model)
 	return []string{
-		"--model", modelSource.ModelPath(),
-		"--served-model-name", modelSource.ModelName(),
+		"--model", source.ModelPath(),
+		"--served-model-name", source.ModelName(),
 		"--port", strconv.Itoa(pkg.DEFAULT_BACKEND_PORT),
 	}
 }
diff --git a/...ntroller_helper/model_source/model_hub.go → ...ontroller_helper/model_source/modelhub.go b/...ntroller_helper/model_source/model_hub.go → ...ontroller_helper/model_source/modelhub.go
@@ -25,7 +25,7 @@ import (
 	"k8s.io/utils/ptr"
 )
 
-var _ DataSourceProvider = &ModelHubProvider{}
+var _ ModelSourceProvider = &ModelHubProvider{}
 
 type ModelHubProvider struct {
 	model *coreapi.Model

diff --git a/pkg/controller_helper/model_source/modelsource.go b/pkg/controller_helper/model_source/modelsource.go
@@ -22,13 +22,13 @@ import (
 	coreapi "inftyai.com/llmaz/api/core/v1alpha1"
 )
 
-type DataSourceProvider interface {
+type ModelSourceProvider interface {
 	ModelName() string
 	ModelPath() string
 	InjectModelLoader(*corev1.PodTemplateSpec)
 }
 
-func NewDataSourceProvider(model *coreapi.Model) DataSourceProvider {
+func NewModelSourceProvider(model *coreapi.Model) ModelSourceProvider {
 	if model.Spec.Source.ModelHub != nil {
 		return &ModelHubProvider{model: model}
 	}

diff --git a/pkg/controller_helper/model_source/modelsource_test.go b/pkg/controller_helper/model_source/modelsource_test.go
@@ -46,7 +46,7 @@ func Test_ModelSourceProvider(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			provider := NewDataSourceProvider(tc.model)
+			provider := NewModelSourceProvider(tc.model)
 			if tc.wantModelName != provider.ModelName() {
 				t.Fatalf("unexpected model name, want %s, got %s", tc.wantModelName, provider.ModelName())
 			}

diff --git a/test/e2e/suit_test.go b/test/e2e/suit_test.go
@@ -34,7 +34,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	lws "sigs.k8s.io/lws/api/leaderworkerset/v1"
 
-	api "inftyai.com/llmaz/api/core/v1alpha1"
 	coreapi "inftyai.com/llmaz/api/core/v1alpha1"
 	inferenceapi "inftyai.com/llmaz/api/inference/v1alpha1"
 	"inftyai.com/llmaz/test/util"
@@ -103,6 +102,6 @@ func readyForTesting(client client.Client) {
 	// Delete this model before beginning tests.
 	Expect(client.Delete(ctx, model))
 	Eventually(func() error {
-		return client.Get(ctx, types.NamespacedName{Name: model.Name, Namespace: model.Namespace}, &api.Model{})
+		return client.Get(ctx, types.NamespacedName{Name: model.Name, Namespace: model.Namespace}, &coreapi.Model{})
 	}).ShouldNot(Succeed())
 }