Merge pull request #701 from revit13/kfphap

Add KFP workflow for HAP.

.github/workflows/test-universal-hap-kfp.yml

@@ -0,0 +1,116 @@
+#
+# DO NOT EDIT THIS FILE: it is generated from test-transform.template,  Edit there and run make to change these files
+#
+name: Test KFP - transforms/universal/hap
+
+on:
+    workflow_dispatch:
+    push:
+        branches:
+            - "dev"
+            - "releases/**"
+        tags:
+            - "*"
+        paths:
+            - ".make.*"
+            - "transforms/.make.workflows"
+            - "transforms/universal/hap/**"
+            - "!kfp/**" # This is tested in separate workflow
+            - "!data-processing-lib/**" # This is tested in separate workflow
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
+    pull_request:
+        branches:
+            - "dev"
+            - "releases/**"
+        paths:
+            - ".make.*"
+            - "transforms/.make.workflows"
+            - "transforms/universal/hap/**"
+            - "!data-processing-lib/**" # This is tested in separate workflow
+            - "!kfp/**" # This is tested in separate workflow
+            - "!**.md"
+            - "!**/doc/**"
+            - "!**/images/**"
+            - "!**.gitignore"
+
+# taken from https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre
+concurrency:
+    group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+    cancel-in-progress: true
+
+jobs:
+    test-kfp-v1:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+            - name: Free up space in github runner
+              # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+              run: |
+                  df -h
+                  sudo rm -rf "/usr/local/share/boost"
+                  sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+                  sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
+                  sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
+                  df -h
+            - name: Import environment variables
+              run: |
+                  cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV
+                  echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV
+                  echo "REPOROOT=$PWD" >> $GITHUB_ENV
+                  echo "PATH=$PATH:/tmp" >> $GITHUB_ENV
+            - name: Test V1 KFP workflow for transforms/universal/hap
+              timeout-minutes: 120
+              run: |
+                  KFP_BLACK_LIST=$(./scripts/check-workflows.sh -show-kfp-black-list)
+                  if [ -e "transforms/universal/hap/Makefile" -a -e "transforms/universal/hap/kfp_ray/Makefile" ]; then
+                     transform=$(basename "transforms/universal/hap")
+                     if echo ${KFP_BLACK_LIST} | grep -qv ${transform}; then
+                        $PWD/scripts/workflow_helper.sh install-tools
+                        $PWD/scripts/workflow_helper.sh test-workflow transforms/universal/hap   
+                     else
+                        $PWD/scripts/workflow_helper.sh build-workflow transforms/universal/hap
+                     fi              
+                  else
+                     echo "Skipping transforms/universal/hap kfp test for lack of Makefile and/or kfp_ray/Makefile"
+                   fi
+
+    test-kfp-v2:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v4
+            - name: Free up space in github runner
+              # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173
+              run: |
+                  df -h
+                  sudo rm -rf "/usr/local/share/boost"
+                  sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+                  sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
+                  sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
+                  df -h
+            - name: Import environment variables
+              run: |
+                  cat scripts/k8s-setup/requirements.env >> $GITHUB_ENV
+                  echo "K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup" >> $GITHUB_ENV
+                  echo "REPOROOT=$PWD" >> $GITHUB_ENV
+                  echo "PATH=$PATH:/tmp" >> $GITHUB_ENV
+                  echo "KFPv2=1" >> $GITHUB_ENV
+            - name: Test V2 KFP workflow for transforms/universal/hap
+              timeout-minutes: 120
+              run: |
+                  KFP_BLACK_LIST=$(./scripts/check-workflows.sh -show-kfp-black-list)
+                  if [ -e "transforms/universal/hap/Makefile" -a -e "transforms/universal/hap/kfp_ray/Makefile" ]; then
+                     transform=$(basename "transforms/universal/hap")
+                     if echo ${KFP_BLACK_LIST} | grep -qv ${transform}; then
+                        $PWD/scripts/workflow_helper.sh install-tools
+                        $PWD/scripts/workflow_helper.sh test-workflow transforms/universal/hap   
+                     else
+                        $PWD/scripts/workflow_helper.sh build-workflow transforms/universal/hap
+                     fi              
+                  else
+                     echo "Skipping transforms/universal/hap kfp test for lack of Makefile and/or kfp_ray/Makefile"
+                   fi

scripts/k8s-setup/populate_minio.sh

@@ -42,4 +42,4 @@ mc cp --recursive ${REPOROOT}/transforms/universal/noop/ray/test-data/input/ kfp
 mc cp --recursive ${REPOROOT}/transforms/universal/tokenization/ray/test-data/ds01/input/ kfp/test/tokenization/ds01/input
 mc cp --recursive ${REPOROOT}/transforms/universal/profiler/ray/test-data/input/ kfp/test/profiler/input
 mc cp --recursive ${REPOROOT}/transforms/universal/resize/ray/test-data/input/ kfp/test/resize/input
-
+mc cp --recursive ${REPOROOT}/transforms/universal/hap/ray/test-data/input/ kfp/test/hap/input

transforms/universal/hap/kfp_ray/Makefile

@@ -0,0 +1,59 @@
+REPOROOT=${CURDIR}/../../../../
+
+WORKFLOW_VENV_ACTIVATE=${REPOROOT}/transforms/venv/bin/activate
+include $(REPOROOT)/transforms/.make.workflows
+
+# Include the common configuration for this transform
+include ../transform.config
+
+SRC_DIR=${CURDIR}/../ray/
+
+PYTHON_WF := $(shell find ./ -name '*_wf.py')
+YAML_WF := $(patsubst %.py, %.yaml, ${PYTHON_WF})
+
+workflow-venv: .check_python_version ${WORKFLOW_VENV_ACTIVATE}
+
+.PHONY: clean
+clean:
+	@# Help: Clean up the virtual environment.
+	rm -rf ${REPOROOT}/transforms/venv 
+
+venv::
+
+build::
+
+setup::
+
+test::
+
+test-src::
+
+test-image::
+
+publish::
+
+image::
+
+kind-load-image::
+
+docker-load-image::
+
+docker-save-image::
+
+.PHONY: workflow-build
+workflow-build: workflow-venv
+	$(MAKE) $(YAML_WF)
+
+.PHONY: workflow-test
+workflow-test: workflow-build
+	$(MAKE) .workflows.test-pipeline TRANSFORM_SRC=${SRC_DIR} PIPELINE_FILE=hap_wf.yaml
+
+.PHONY: workflow-upload
+workflow-upload: workflow-build
+	@for file in $(YAML_WF); do \
+		$(MAKE) .workflows.upload-pipeline PIPELINE_FILE=$$file; \
+	done
+
+.PHONY: workflow-generate
+workflow-generate: workflow-venv
+	. ${WORKFLOW_VENV_ACTIVATE}  && ../../../../kfp/pipeline_generator/single-pipeline/run.sh -c `pwd`/pipeline_definitions.yaml -od  .