Local model upload to PVC

README.md

@@ -145,6 +145,31 @@ Everything should be shown green. If it is not, click the icon of the faulty obj
   * Edit contents of [thanos-secret](acm/odh-core/acm-observability/secrets/thanos.yaml) file.
   * Install the ACM observability stack by running `make install`
 
+### Using local models in pipelines
+
+In `pipelines/tekton/azureml-container-pipeline/model-upload/` you can upload a local 
+model file to be used in our pipelines. This is done by uploading a model to a PVC
+and copying that model to our pipeline's workspace for use while it is running.
+
+Upload model to PVC:
+```bash
+make MODEL_PATH="PATH_TO_A_FILE" NAME=my-model create
+```
+You should get a final output showing details of the upload
+```
+PVC name: model-upload-pvc
+Size: 1G
+Model path in pod: /workspace/model-upload-pvc/model_dir/model.model
+```
+You can set the `SIZE` and `PVC` values aswell
+```bash
+make MODEL_PATH="PATH_TO_A_FILE" NAME=my-model SIZE=1G PVC=my-new-PVC create
+```
+
+In `pipelines/tekton/build-container-image-pipeline` you can set the 
+`fetch-model` to `pvc` for the pipeline to copy the file from the 
+`model-workspace` which can be set to the PVC created in the last step
+
 ## Contributing
 
 See [CONTRIBUTING.md](CONTRIBUTING.md).

pipelines/tekton/azureml-container-pipeline/model-upload/Makefile

@@ -0,0 +1,30 @@
+# Used for uploading local model files to the cluster through a 
+# pod to a PVC. Use the create target to create the pod and PVC
+# needed and upload the file
+
+SIZE=1G
+PVC=model-upload-pvc
+
+# Creates a PVC and a pod which mounts the PVC created
+# then uploads the file given
+# SIZE - size of the storage used in the PVC created
+# PVC - name of the PVC 
+# MODEL_PATH - local path to model file to upload 
+# NAME - name of the file when uploaded to the pod
+.PHONY: create
+create:
+# create pod and PVC and wait for pod to be ready
+	@oc process -f local-model-pvc-template.yaml -p PVC=${PVC} -p SIZE=${SIZE} | oc create -f -
+	@oc process -f local-model-to-pvc-pod-template.yaml -p PVC=${PVC} | oc create -f -
+	@oc wait pod local-model-to-pvc-pod --for condition=Ready=True --timeout=60s
+
+# upload model to the pod to the PVC
+	@oc exec local-model-to-pvc-pod -- mkdir workspace/${PVC}/model_dir/ -p
+	@oc cp ${MODEL_PATH} local-model-to-pvc-pod:/workspace/${PVC}/model_dir/${NAME}
+
+# delete pod
+	@oc delete pod local-model-to-pvc-pod
+
+	@echo "PVC name: ${PVC}" 
+	@echo "Size: ${SIZE}" 
+	@echo "Model path in pod: /workspace/${PVC}/model_dir/${NAME}" 

pipelines/tekton/azureml-container-pipeline/model-upload/local-model-pvc-template.yaml

@@ -0,0 +1,22 @@
+apiVersion: template.openshift.io/v1
+kind: Template
+metadata:
+  name: local-model-pvc-template
+objects:
+- apiVersion: v1
+  kind: PersistentVolumeClaim
+  metadata:
+    name: ${PVC}
+  spec:
+    accessModes:
+    - ReadWriteOnce
+    resources:
+      requests:
+        storage: ${SIZE}
+parameters:
+- name: PVC
+  description: Name of PVC to be mounted to pod
+  value: local-model
+- name: SIZE
+  description: Size of the PVC
+  value: 1Gi

pipelines/tekton/azureml-container-pipeline/model-upload/local-model-to-pvc-pod-template.yaml

@@ -0,0 +1,28 @@
+apiVersion: template.openshift.io/v1
+kind: Template
+metadata:
+  name: local-model-to-pvc-pod-template
+objects:
+- apiVersion: v1
+  kind: Pod
+  metadata:
+    name: local-model-to-pvc-pod
+  spec:
+    volumes:
+      - name: ${PVC}
+        persistentVolumeClaim:
+          claimName: ${PVC}
+    containers:
+    - name: local-model-to-pvc-container
+      image: ubi9
+      stdin: true
+      tty: true
+      securityContext:
+        allowPrivilegeEscalation: false
+      volumeMounts:
+          - mountPath: /workspace/${PVC}
+            name: ${PVC}
+parameters:
+- name: PVC
+  description: Name of PVC to be mounted to pod
+  value: local-model

pipelines/tekton/build-container-image-pipeline/build-container-image-pipeline.yaml

@@ -30,6 +30,22 @@ spec:
     type: string
     default: "main"
   tasks:
+  - name: copy-model-from-pvc
+    taskRef:
+      kind: Task
+      name: copy-model-from-pvc
+    params:
+    - name: model-name
+      value: $(params.model-name)
+    workspaces:
+    - name: source-workspace
+      workspace: model-workspace
+    - name: destination-workspace
+      workspace: buildah-cache
+    when:
+    - input: "$(params.fetch-model)"
+      operator: in
+      values: ["pvc"]
   - name: git-clone-model-repo
     taskRef:
       kind: ClusterTask
@@ -149,6 +165,7 @@ spec:
       workspace: buildah-cache
   workspaces:
   - name: buildah-cache
+  - name: model-workspace
   - name: aws-secret
   - name: git-basic-auth
     optional: true

pipelines/tekton/build-container-image-pipeline/build-container-image-pipelinerun-bike-rentals.yaml

@@ -33,6 +33,9 @@ spec:
   serviceAccountName: pipeline
   timeout: 1h0m0s
   workspaces:
+  - name: model-workspace
+    persistentVolumeClaim:
+      claimName: basic-pvc
   - name: buildah-cache
     persistentVolumeClaim:
       claimName: buildah-cache-pvc

pipelines/tekton/build-container-image-pipeline/copy-model-from-pvc.yaml

@@ -0,0 +1,30 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: copy-model-from-pvc
+spec:
+  description: This Task can be used to copy a model from another pvc
+  params:
+  - name: model-name
+    type: string
+  steps:
+  - name: download-model-s3
+    image: quay.io/opendatahub/kserve-storage-initializer:v0.11
+    script: |
+      SOURCE_PATH="$(workspaces.source-workspace.path)/model_dir/$(params.model-name)"
+
+      DEST_PATH="$(workspaces.destination-workspace.path)/model_dir/$(params.model-name)"
+
+      echo "Copying model file $SOURCE_PATH"
+      echo "to $DEST_PATH"
+
+      DIR_PATH="$(dirname $(workspaces.destination-workspace.path)/model_dir/$(params.model-name))"
+
+      mkdir -p $DIR_PATH
+
+      cp $SOURCE_PATH $DEST_PATH 
+  workspaces:
+  - description: The workspace the model is being copied from
+    name: source-workspace
+  - description: The workspace the model is being copied to
+    name: destination-workspace

pipelines/tekton/build-container-image-pipeline/kustomization.yaml

@@ -5,3 +5,4 @@ resources:
 - kserve-download-model.yaml
 - check-model-and-containerfile-exists.yaml
 - build-container-image-pipeline.yaml
+- copy-model-from-pvc.yaml