diff --git a/README.md b/README.md index 0227e103..11ebac9b 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,31 @@ Everything should be shown green. If it is not, click the icon of the faulty obj * Edit contents of [thanos-secret](acm/odh-core/acm-observability/secrets/thanos.yaml) file. * Install the ACM observability stack by running `make install` +### Using local models in pipelines + +In `pipelines/tekton/azureml-container-pipeline/model-upload/` you can upload a local +model file to be used in our pipelines. This is done by uploading a model to a PVC +and copying that model to our pipeline's workspace for use while it is running. + +Upload model to PVC: +```bash +make MODEL_PATH="PATH_TO_A_FILE" NAME=my-model create +``` +You should get a final output showing details of the upload +``` +PVC name: model-upload-pvc +Size: 1G +Model path in pod: /workspace/model-upload-pvc/model_dir/model.model +``` +You can set the `SIZE` and `PVC` values aswell +```bash +make MODEL_PATH="PATH_TO_A_FILE" NAME=my-model SIZE=1G PVC=my-new-PVC create +``` + +In `pipelines/tekton/build-container-image-pipeline` you can set the +`fetch-model` to `pvc` for the pipeline to copy the file from the +`model-workspace` which can be set to the PVC created in the last step + ## Contributing See [CONTRIBUTING.md](CONTRIBUTING.md). diff --git a/pipelines/tekton/azureml-container-pipeline/model-upload/Makefile b/pipelines/tekton/azureml-container-pipeline/model-upload/Makefile new file mode 100644 index 00000000..8ae92137 --- /dev/null +++ b/pipelines/tekton/azureml-container-pipeline/model-upload/Makefile @@ -0,0 +1,30 @@ +# Used for uploading local model files to the cluster through a +# pod to a PVC. Use the create target to create the pod and PVC +# needed and upload the file + +SIZE=1G +PVC=model-upload-pvc + +# Creates a PVC and a pod which mounts the PVC created +# then uploads the file given +# SIZE - size of the storage used in the PVC created +# PVC - name of the PVC +# MODEL_PATH - local path to model file to upload +# NAME - name of the file when uploaded to the pod +.PHONY: create +create: +# create pod and PVC and wait for pod to be ready + @oc process -f local-model-pvc-template.yaml -p PVC=${PVC} -p SIZE=${SIZE} | oc create -f - + @oc process -f local-model-to-pvc-pod-template.yaml -p PVC=${PVC} | oc create -f - + @oc wait pod local-model-to-pvc-pod --for condition=Ready=True --timeout=60s + +# upload model to the pod to the PVC + @oc exec local-model-to-pvc-pod -- mkdir workspace/${PVC}/model_dir/ -p + @oc cp ${MODEL_PATH} local-model-to-pvc-pod:/workspace/${PVC}/model_dir/${NAME} + +# delete pod + @oc delete pod local-model-to-pvc-pod + + @echo "PVC name: ${PVC}" + @echo "Size: ${SIZE}" + @echo "Model path in pod: /workspace/${PVC}/model_dir/${NAME}" diff --git a/pipelines/tekton/azureml-container-pipeline/model-upload/local-model-pvc-template.yaml b/pipelines/tekton/azureml-container-pipeline/model-upload/local-model-pvc-template.yaml new file mode 100644 index 00000000..745008b9 --- /dev/null +++ b/pipelines/tekton/azureml-container-pipeline/model-upload/local-model-pvc-template.yaml @@ -0,0 +1,22 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: local-model-pvc-template +objects: +- apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: ${PVC} + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: ${SIZE} +parameters: +- name: PVC + description: Name of PVC to be mounted to pod + value: local-model +- name: SIZE + description: Size of the PVC + value: 1Gi diff --git a/pipelines/tekton/azureml-container-pipeline/model-upload/local-model-to-pvc-pod-template.yaml b/pipelines/tekton/azureml-container-pipeline/model-upload/local-model-to-pvc-pod-template.yaml new file mode 100644 index 00000000..465910c7 --- /dev/null +++ b/pipelines/tekton/azureml-container-pipeline/model-upload/local-model-to-pvc-pod-template.yaml @@ -0,0 +1,28 @@ +apiVersion: template.openshift.io/v1 +kind: Template +metadata: + name: local-model-to-pvc-pod-template +objects: +- apiVersion: v1 + kind: Pod + metadata: + name: local-model-to-pvc-pod + spec: + volumes: + - name: ${PVC} + persistentVolumeClaim: + claimName: ${PVC} + containers: + - name: local-model-to-pvc-container + image: ubi9 + stdin: true + tty: true + securityContext: + allowPrivilegeEscalation: false + volumeMounts: + - mountPath: /workspace/${PVC} + name: ${PVC} +parameters: +- name: PVC + description: Name of PVC to be mounted to pod + value: local-model diff --git a/pipelines/tekton/build-container-image-pipeline/build-container-image-pipeline.yaml b/pipelines/tekton/build-container-image-pipeline/build-container-image-pipeline.yaml index 9059e79d..6c5a873d 100644 --- a/pipelines/tekton/build-container-image-pipeline/build-container-image-pipeline.yaml +++ b/pipelines/tekton/build-container-image-pipeline/build-container-image-pipeline.yaml @@ -30,6 +30,22 @@ spec: type: string default: "main" tasks: + - name: copy-model-from-pvc + taskRef: + kind: Task + name: copy-model-from-pvc + params: + - name: model-name + value: $(params.model-name) + workspaces: + - name: source-workspace + workspace: model-workspace + - name: destination-workspace + workspace: buildah-cache + when: + - input: "$(params.fetch-model)" + operator: in + values: ["pvc"] - name: git-clone-model-repo taskRef: kind: ClusterTask @@ -149,6 +165,7 @@ spec: workspace: buildah-cache workspaces: - name: buildah-cache + - name: model-workspace - name: aws-secret - name: git-basic-auth optional: true diff --git a/pipelines/tekton/build-container-image-pipeline/build-container-image-pipelinerun-bike-rentals.yaml b/pipelines/tekton/build-container-image-pipeline/build-container-image-pipelinerun-bike-rentals.yaml index ab1d88be..91622915 100644 --- a/pipelines/tekton/build-container-image-pipeline/build-container-image-pipelinerun-bike-rentals.yaml +++ b/pipelines/tekton/build-container-image-pipeline/build-container-image-pipelinerun-bike-rentals.yaml @@ -33,6 +33,9 @@ spec: serviceAccountName: pipeline timeout: 1h0m0s workspaces: + - name: model-workspace + persistentVolumeClaim: + claimName: basic-pvc - name: buildah-cache persistentVolumeClaim: claimName: buildah-cache-pvc diff --git a/pipelines/tekton/build-container-image-pipeline/copy-model-from-pvc.yaml b/pipelines/tekton/build-container-image-pipeline/copy-model-from-pvc.yaml new file mode 100644 index 00000000..793c2b1b --- /dev/null +++ b/pipelines/tekton/build-container-image-pipeline/copy-model-from-pvc.yaml @@ -0,0 +1,30 @@ +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: copy-model-from-pvc +spec: + description: This Task can be used to copy a model from another pvc + params: + - name: model-name + type: string + steps: + - name: download-model-s3 + image: quay.io/opendatahub/kserve-storage-initializer:v0.11 + script: | + SOURCE_PATH="$(workspaces.source-workspace.path)/model_dir/$(params.model-name)" + + DEST_PATH="$(workspaces.destination-workspace.path)/model_dir/$(params.model-name)" + + echo "Copying model file $SOURCE_PATH" + echo "to $DEST_PATH" + + DIR_PATH="$(dirname $(workspaces.destination-workspace.path)/model_dir/$(params.model-name))" + + mkdir -p $DIR_PATH + + cp $SOURCE_PATH $DEST_PATH + workspaces: + - description: The workspace the model is being copied from + name: source-workspace + - description: The workspace the model is being copied to + name: destination-workspace diff --git a/pipelines/tekton/build-container-image-pipeline/kustomization.yaml b/pipelines/tekton/build-container-image-pipeline/kustomization.yaml index dd5dca25..dd40b003 100644 --- a/pipelines/tekton/build-container-image-pipeline/kustomization.yaml +++ b/pipelines/tekton/build-container-image-pipeline/kustomization.yaml @@ -5,3 +5,4 @@ resources: - kserve-download-model.yaml - check-model-and-containerfile-exists.yaml - build-container-image-pipeline.yaml +- copy-model-from-pvc.yaml