From 8836390d63f0144f2e6ce7a3110a8a1e2235d953 Mon Sep 17 00:00:00 2001 From: Revital Sur Date: Sun, 22 Sep 2024 14:03:26 +0300 Subject: [PATCH 1/6] Fix kfp pipelines testing in github workflow. Signed-off-by: Revital Sur --- .github/workflows/Makefile | 29 +- .github/workflows/README.md | 36 +- .github/workflows/build-library.yml.old | 59 --- .github/workflows/runs.h | 3 + .../workflows/test-code-code2parquet-kfp.yml | 114 +++++ .../workflows/test-code-code_quality-kfp.yml | 114 +++++ .../test-code-header_cleanser-kfp.yml | 114 +++++ .../test-code-inputcode2parquet-kfp.yml | 114 +++++ .github/workflows/test-code-malware-kfp.yml | 114 +++++ .../test-code-proglang_select-kfp.yml | 114 +++++ .../test-code-repo_level_ordering-kfp.yml | 114 +++++ .github/workflows/test-kfp-transform.template | 114 +++++ .github/workflows/test-kfp.yml | 2 + .../test-language-doc_quality-kfp.yml | 114 +++++ .../workflows/test-language-lang_id-kfp.yml | 114 +++++ .../test-language-text_encoder-kfp.yml | 114 +++++ .../workflows/test-universal-doc_id-kfp.yml | 114 +++++ .../workflows/test-universal-ededup-kfp.yml | 114 +++++ .../workflows/test-universal-fdedup-kfp.yml | 114 +++++ .../workflows/test-universal-filter-kfp.yml | 114 +++++ .github/workflows/test-universal-noop-kfp.yml | 114 +++++ .../workflows/test-universal-profiler-kfp.yml | 114 +++++ .../workflows/test-universal-resize-kfp.yml | 114 +++++ .../test-universal-tokenization-kfp.yml | 114 +++++ .github/workflows/test.yml.old | 405 ------------------ data-processing-lib/ray/README.md | 2 +- kfp/kfp_ray_components/README.md | 2 +- kfp/kfp_ray_components/src/execute_ray_job.py | 2 +- 28 files changed, 2216 insertions(+), 490 deletions(-) delete mode 100644 .github/workflows/build-library.yml.old create mode 100755 .github/workflows/runs.h create mode 100644 .github/workflows/test-code-code2parquet-kfp.yml create mode 100644 .github/workflows/test-code-code_quality-kfp.yml create mode 100644 .github/workflows/test-code-header_cleanser-kfp.yml create mode 100644 .github/workflows/test-code-inputcode2parquet-kfp.yml create mode 100644 .github/workflows/test-code-malware-kfp.yml create mode 100644 .github/workflows/test-code-proglang_select-kfp.yml create mode 100644 .github/workflows/test-code-repo_level_ordering-kfp.yml create mode 100644 .github/workflows/test-kfp-transform.template create mode 100644 .github/workflows/test-language-doc_quality-kfp.yml create mode 100644 .github/workflows/test-language-lang_id-kfp.yml create mode 100644 .github/workflows/test-language-text_encoder-kfp.yml create mode 100644 .github/workflows/test-universal-doc_id-kfp.yml create mode 100644 .github/workflows/test-universal-ededup-kfp.yml create mode 100644 .github/workflows/test-universal-fdedup-kfp.yml create mode 100644 .github/workflows/test-universal-filter-kfp.yml create mode 100644 .github/workflows/test-universal-noop-kfp.yml create mode 100644 .github/workflows/test-universal-profiler-kfp.yml create mode 100644 .github/workflows/test-universal-resize-kfp.yml create mode 100644 .github/workflows/test-universal-tokenization-kfp.yml delete mode 100644 .github/workflows/test.yml.old diff --git a/.github/workflows/Makefile b/.github/workflows/Makefile index d8da3e720..8a17ffa08 100644 --- a/.github/workflows/Makefile +++ b/.github/workflows/Makefile @@ -7,20 +7,41 @@ CODE_TRANSFORMS=code2parquet code_quality header_cleanser malware proglang_selec LANG_TRANSFORMS=doc_chunk doc_quality lang_id pdf2parquet pii_redactor text_encoder +# A list that holds transforms that should not be tested with KFP +KFP_BLACK_LIST="doc_chunk,pdf2parquet,pii_redactor" + transform-tests: - $(MAKE) TRANSFORM_SUBDIR=universal .transform-tests - $(MAKE) TRANSFORM_SUBDIR=language .transform-tests - $(MAKE) TRANSFORM_SUBDIR=code .transform-tests + $(MAKE) TRANSFORM_SUBDIR=universal .transform-tests .transform-kfp-tests + $(MAKE) TRANSFORM_SUBDIR=universal .transform-kfp-tests + $(MAKE) TRANSFORM_SUBDIR=language .transform-tests + $(MAKE) TRANSFORM_SUBDIR=language .transform-kfp-tests + $(MAKE) TRANSFORM_SUBDIR=code .transform-tests + $(MAKE) TRANSFORM_SUBDIR=code .transform-kfp-tests # Expects # TRANSFORM_SUBDIR transforms subdirectory (such as universal) .transform-tests: - @for i in $$(find ../../transforms/$(TRANSFORM_SUBDIR) -depth 1 -type d); do \ + @for i in $$(find ../../transforms/$(TRANSFORM_SUBDIR) -mindepth 1 -maxdepth 1 -type d); do \ dir=$$(basename $$i); \ yml=test-$(TRANSFORM_SUBDIR)-$$dir.yml; \ echo Generating $$yml; \ cat test-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/$${TRANSFORM_SUBDIR}/$$dir?g" > $$yml; \ done +.transform-kfp-tests: + @for i in $$(find ../../transforms/$(TRANSFORM_SUBDIR) -mindepth 1 -maxdepth 1 -type d); do \ + dir=$$(basename $$i); \ + z=$$(echo ${KFP_BLACK_LIST} | grep -v $$dir); \ + if [ ! -d ../../transforms/$(TRANSFORM_SUBDIR)/$$dir/kfp_ray ] || [ -z "$$z" ]; then \ + continue; \ + fi; \ + yml=test-$(TRANSFORM_SUBDIR)-$$dir-kfp.yml; \ + echo Generating $$yml; \ + cat test-kfp-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/$${TRANSFORM_SUBDIR}/$$dir?g" > $$yml; \ + done + + + + diff --git a/.github/workflows/README.md b/.github/workflows/README.md index faf8139b4..d4a09b1c3 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -1,20 +1,23 @@ # Workflow Management -Here we have the start of a system to automatically generated github workflows (currently only for transforms). +Here we have the start of a system to automatically generated github workflows. In general, the design is to use templates and `make` to generate/update the workflows. #### Goals 1. Run only tests for a given transform when only the transform changes. Includes python, ray, spark and kfp_ray as available. 2. When the core dpk lib components files changes, test all transforms -3. When the shared kfp components changes, test a randomly selected transform test - (We would like to avoid running all transform kfp tests in one PR) +3. When the shared kfp components changes or core dpk lib components files changes, + test a randomly selected transform test. Otherwise run kfp test for the changed transforms. 4. Extra credit: If .md or other non-code changes are made, run no tests. #### Assumptions 1. All transforms will have test workflows. A transform can disable its tests locally (temporarily?) by renaming its Makefile. For example, `cp transforms/universal/noop/Makefile transforms/universal/noop/Makefile.disabled`. +A github action for kfp testing will not be generated if it appears in `KFP_BLACK_LIST` +in the [Makefile](./Makefile). + ## DPK libraries (`data-processing-lib` directory) The DPK libraries, in data-processing-lib/{python,ray,spark}, are tested @@ -26,18 +29,18 @@ The transforms test workflows also depend on this directory tree and so changes made here will trigger transform tests. ## Transforms (`transforms` directory tree) -We define a unique test workflow for each transform, based on a common -template [test-transform.template](test-transform.template). -The [Makefile](Makefile) is used to (re)generate all workflows a necessary. -By design, workflows for a given transform should run when +We define two test workflows for each transform: one is based on a common +template [test-transform.template](test-transform.template) and the other, for kfp testing, +is based on a common template [test-kfp-transform.template](test-kfp-transform.template). +The [Makefile](Makefile) is used to (re)generate all workflows as necessary. +By design, non kfp workflows for a given transform should run when * anything of substance effecting operation is modified in the transform's directory tree. * anything in the core libraries in this repo (e.g., data-processing/lib) assuming the transform depends on these. -Note that the kfp tests (in kfp_ray/Makefile workflow-test) for a given transform are -**not** currently being run when the transform's tests are run. -Currently these are run randomly via the [test-kfp.yml](test-kfp.yml). -We expect to fix this is in the future. +The generated kfp workflows should run when anything of substance effecting operation is modified in the transform's directory tree +and non of the core libraries in this repo nor the kfp components were changed. +Otherwise a randomly chosen transform will undergo KFP testing, triggered by the [test-kfp.yml](test-kfp.yml) workflow. When a new transform is added to the repository, @@ -58,16 +61,11 @@ git push --set-upstream origin new-branch Like DPK core libs, kfp tests are defined in [test-kfp.yml](test-kfp.yml) and run whenever changes are made in -the `kfp` directory tree. Tests currently include - -1. test kfp on randomly selected transform. - -Eventually we would like to enable the transform-specific kfp test -when only the transform code is modified or maybe when only -the `kfp_ray` directory contents is modified. +the `kfp` directory tree as well as in the DPK core libs. Tests currently include +test kfp on randomly selected transform. ## Miscellaneous [test-misc.yml](test-misc.yml) defines some repo consistency tests including 1. Make sure `set-versions` make target can be run recursively throughout the repo -2. Makes sure there is a test workflow for each transform in the repo. \ No newline at end of file +2. Makes sure there is a test workflow for each transform in the repo. diff --git a/.github/workflows/build-library.yml.old b/.github/workflows/build-library.yml.old deleted file mode 100644 index d7219ea34..000000000 --- a/.github/workflows/build-library.yml.old +++ /dev/null @@ -1,59 +0,0 @@ -name: Build Library - -on: - workflow_dispatch: - push: - branches: - - "dev" - - "releases/**" - paths: - - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" - pull_request: - branches: - - "dev" - - "releases/**" - paths: - - "data-processing-lib/**" - - "!**.md" - - "!**/doc/**" - - "!**/.gitignore" -jobs: - build-python-lib: - runs-on: ubuntu-22.04 - strategy: - matrix: - python: - - "3.11" - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build data-processing-lib/python - run: | - make -C data-processing-lib/python DOCKER=docker venv build - build-ray-lib: - runs-on: ubuntu-22.04 - strategy: - matrix: - python: - - "3.11" - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build data-processing-lib/ray - run: | - make -C data-processing-lib/ray DOCKER=docker venv build - build-spark-lib: - runs-on: ubuntu-22.04 - strategy: - matrix: - python: - - "3.11" - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build data-processing-lib/spark - run: | - make -C data-processing-lib/spark DOCKER=docker venv build diff --git a/.github/workflows/runs.h b/.github/workflows/runs.h new file mode 100755 index 000000000..5a54bb616 --- /dev/null +++ b/.github/workflows/runs.h @@ -0,0 +1,3 @@ + +echo ${PWD} +find ../../transforms/universal -mindepth 1 -maxdepth 1 -type d diff --git a/.github/workflows/test-code-code2parquet-kfp.yml b/.github/workflows/test-code-code2parquet-kfp.yml new file mode 100644 index 000000000..41f58e0cb --- /dev/null +++ b/.github/workflows/test-code-code2parquet-kfp.yml @@ -0,0 +1,114 @@ +# +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files +# +name: Test KFP - transforms/code/code2parquet + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/code/code2parquet/**" + - "!kfp/**" # This is tested in separate workflow + - "!data-processing-lib/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/code/code2parquet/**" + - "!data-processing-lib/**" # This is tested in separate workflow + - "!kfp/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + + +jobs: + test-kfp-v1: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/code2parquet workflow-test + echo "Run transforms/code/code2parquet completed" + + test-kfp-v2: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/code2parquet workflow-test + header_text "Run transforms/code/code2parquet completed" diff --git a/.github/workflows/test-code-code_quality-kfp.yml b/.github/workflows/test-code-code_quality-kfp.yml new file mode 100644 index 000000000..21fa63296 --- /dev/null +++ b/.github/workflows/test-code-code_quality-kfp.yml @@ -0,0 +1,114 @@ +# +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files +# +name: Test KFP - transforms/code/code_quality + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/code/code_quality/**" + - "!kfp/**" # This is tested in separate workflow + - "!data-processing-lib/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/code/code_quality/**" + - "!data-processing-lib/**" # This is tested in separate workflow + - "!kfp/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + + +jobs: + test-kfp-v1: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/code_quality workflow-test + echo "Run transforms/code/code_quality completed" + + test-kfp-v2: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/code_quality workflow-test + header_text "Run transforms/code/code_quality completed" diff --git a/.github/workflows/test-code-header_cleanser-kfp.yml b/.github/workflows/test-code-header_cleanser-kfp.yml new file mode 100644 index 000000000..25f54b528 --- /dev/null +++ b/.github/workflows/test-code-header_cleanser-kfp.yml @@ -0,0 +1,114 @@ +# +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files +# +name: Test KFP - transforms/code/header_cleanser + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/code/header_cleanser/**" + - "!kfp/**" # This is tested in separate workflow + - "!data-processing-lib/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/code/header_cleanser/**" + - "!data-processing-lib/**" # This is tested in separate workflow + - "!kfp/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + + +jobs: + test-kfp-v1: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/header_cleanser workflow-test + echo "Run transforms/code/header_cleanser completed" + + test-kfp-v2: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/header_cleanser workflow-test + header_text "Run transforms/code/header_cleanser completed" diff --git a/.github/workflows/test-code-inputcode2parquet-kfp.yml b/.github/workflows/test-code-inputcode2parquet-kfp.yml new file mode 100644 index 000000000..32ed1ad07 --- /dev/null +++ b/.github/workflows/test-code-inputcode2parquet-kfp.yml @@ -0,0 +1,114 @@ +# +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files +# +name: Test KFP - transforms/code/inputcode2parquet + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/code/inputcode2parquet/**" + - "!kfp/**" # This is tested in separate workflow + - "!data-processing-lib/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/code/inputcode2parquet/**" + - "!data-processing-lib/**" # This is tested in separate workflow + - "!kfp/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + + +jobs: + test-kfp-v1: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/inputcode2parquet workflow-test + echo "Run transforms/code/inputcode2parquet completed" + + test-kfp-v2: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/inputcode2parquet workflow-test + header_text "Run transforms/code/inputcode2parquet completed" diff --git a/.github/workflows/test-code-malware-kfp.yml b/.github/workflows/test-code-malware-kfp.yml new file mode 100644 index 000000000..9bd937f46 --- /dev/null +++ b/.github/workflows/test-code-malware-kfp.yml @@ -0,0 +1,114 @@ +# +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files +# +name: Test KFP - transforms/code/malware + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/code/malware/**" + - "!kfp/**" # This is tested in separate workflow + - "!data-processing-lib/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/code/malware/**" + - "!data-processing-lib/**" # This is tested in separate workflow + - "!kfp/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + + +jobs: + test-kfp-v1: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/malware workflow-test + echo "Run transforms/code/malware completed" + + test-kfp-v2: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/malware workflow-test + header_text "Run transforms/code/malware completed" diff --git a/.github/workflows/test-code-proglang_select-kfp.yml b/.github/workflows/test-code-proglang_select-kfp.yml new file mode 100644 index 000000000..bbe257964 --- /dev/null +++ b/.github/workflows/test-code-proglang_select-kfp.yml @@ -0,0 +1,114 @@ +# +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files +# +name: Test KFP - transforms/code/proglang_select + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/code/proglang_select/**" + - "!kfp/**" # This is tested in separate workflow + - "!data-processing-lib/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/code/proglang_select/**" + - "!data-processing-lib/**" # This is tested in separate workflow + - "!kfp/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + + +jobs: + test-kfp-v1: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/proglang_select workflow-test + echo "Run transforms/code/proglang_select completed" + + test-kfp-v2: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/proglang_select workflow-test + header_text "Run transforms/code/proglang_select completed" diff --git a/.github/workflows/test-code-repo_level_ordering-kfp.yml b/.github/workflows/test-code-repo_level_ordering-kfp.yml new file mode 100644 index 000000000..c26ecda52 --- /dev/null +++ b/.github/workflows/test-code-repo_level_ordering-kfp.yml @@ -0,0 +1,114 @@ +# +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files +# +name: Test KFP - transforms/code/repo_level_ordering + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/code/repo_level_ordering/**" + - "!kfp/**" # This is tested in separate workflow + - "!data-processing-lib/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/code/repo_level_ordering/**" + - "!data-processing-lib/**" # This is tested in separate workflow + - "!kfp/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + + +jobs: + test-kfp-v1: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/repo_level_ordering workflow-test + echo "Run transforms/code/repo_level_ordering completed" + + test-kfp-v2: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/code/repo_level_ordering workflow-test + header_text "Run transforms/code/repo_level_ordering completed" diff --git a/.github/workflows/test-kfp-transform.template b/.github/workflows/test-kfp-transform.template new file mode 100644 index 000000000..434a57238 --- /dev/null +++ b/.github/workflows/test-kfp-transform.template @@ -0,0 +1,114 @@ +# +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files +# +name: Test KFP - @TARGET_TRANSFORM_DIR@ + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "@TARGET_TRANSFORM_DIR@/**" + - "!kfp/**" # This is tested in separate workflow + - "!data-processing-lib/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "@TARGET_TRANSFORM_DIR@/**" + - "!data-processing-lib/**" # This is tested in separate workflow + - "!kfp/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + + +jobs: + test-kfp-v1: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C @TARGET_TRANSFORM_DIR@ workflow-test + echo "Run @TARGET_TRANSFORM_DIR@ completed" + + test-kfp-v2: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C @TARGET_TRANSFORM_DIR@ workflow-test + header_text "Run @TARGET_TRANSFORM_DIR@ completed" diff --git a/.github/workflows/test-kfp.yml b/.github/workflows/test-kfp.yml index f0984c21b..dc8bc2bd5 100644 --- a/.github/workflows/test-kfp.yml +++ b/.github/workflows/test-kfp.yml @@ -10,6 +10,7 @@ on: - "*" paths: - "kfp/**" + - "data-processing-lib/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -20,6 +21,7 @@ on: - "releases/**" paths: - "kfp/**" + - "data-processing-lib/**" - "!**.md" - "!**/doc/**" - "!**/images/**" diff --git a/.github/workflows/test-language-doc_quality-kfp.yml b/.github/workflows/test-language-doc_quality-kfp.yml new file mode 100644 index 000000000..e9f678595 --- /dev/null +++ b/.github/workflows/test-language-doc_quality-kfp.yml @@ -0,0 +1,114 @@ +# +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files +# +name: Test KFP - transforms/language/doc_quality + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/language/doc_quality/**" + - "!kfp/**" # This is tested in separate workflow + - "!data-processing-lib/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/language/doc_quality/**" + - "!data-processing-lib/**" # This is tested in separate workflow + - "!kfp/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + + +jobs: + test-kfp-v1: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/language/doc_quality workflow-test + echo "Run transforms/language/doc_quality completed" + + test-kfp-v2: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/language/doc_quality workflow-test + header_text "Run transforms/language/doc_quality completed" diff --git a/.github/workflows/test-language-lang_id-kfp.yml b/.github/workflows/test-language-lang_id-kfp.yml new file mode 100644 index 000000000..cf3dec397 --- /dev/null +++ b/.github/workflows/test-language-lang_id-kfp.yml @@ -0,0 +1,114 @@ +# +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files +# +name: Test KFP - transforms/language/lang_id + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/language/lang_id/**" + - "!kfp/**" # This is tested in separate workflow + - "!data-processing-lib/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/language/lang_id/**" + - "!data-processing-lib/**" # This is tested in separate workflow + - "!kfp/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + + +jobs: + test-kfp-v1: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/language/lang_id workflow-test + echo "Run transforms/language/lang_id completed" + + test-kfp-v2: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/language/lang_id workflow-test + header_text "Run transforms/language/lang_id completed" diff --git a/.github/workflows/test-language-text_encoder-kfp.yml b/.github/workflows/test-language-text_encoder-kfp.yml new file mode 100644 index 000000000..d90b76820 --- /dev/null +++ b/.github/workflows/test-language-text_encoder-kfp.yml @@ -0,0 +1,114 @@ +# +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files +# +name: Test KFP - transforms/language/text_encoder + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/language/text_encoder/**" + - "!kfp/**" # This is tested in separate workflow + - "!data-processing-lib/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/language/text_encoder/**" + - "!data-processing-lib/**" # This is tested in separate workflow + - "!kfp/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + + +jobs: + test-kfp-v1: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/language/text_encoder workflow-test + echo "Run transforms/language/text_encoder completed" + + test-kfp-v2: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/language/text_encoder workflow-test + header_text "Run transforms/language/text_encoder completed" diff --git a/.github/workflows/test-universal-doc_id-kfp.yml b/.github/workflows/test-universal-doc_id-kfp.yml new file mode 100644 index 000000000..28c1d8717 --- /dev/null +++ b/.github/workflows/test-universal-doc_id-kfp.yml @@ -0,0 +1,114 @@ +# +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files +# +name: Test KFP - transforms/universal/doc_id + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/universal/doc_id/**" + - "!kfp/**" # This is tested in separate workflow + - "!data-processing-lib/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/universal/doc_id/**" + - "!data-processing-lib/**" # This is tested in separate workflow + - "!kfp/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + + +jobs: + test-kfp-v1: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/doc_id workflow-test + echo "Run transforms/universal/doc_id completed" + + test-kfp-v2: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/doc_id workflow-test + header_text "Run transforms/universal/doc_id completed" diff --git a/.github/workflows/test-universal-ededup-kfp.yml b/.github/workflows/test-universal-ededup-kfp.yml new file mode 100644 index 000000000..5d3481e30 --- /dev/null +++ b/.github/workflows/test-universal-ededup-kfp.yml @@ -0,0 +1,114 @@ +# +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files +# +name: Test KFP - transforms/universal/ededup + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/universal/ededup/**" + - "!kfp/**" # This is tested in separate workflow + - "!data-processing-lib/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/universal/ededup/**" + - "!data-processing-lib/**" # This is tested in separate workflow + - "!kfp/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + + +jobs: + test-kfp-v1: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/ededup workflow-test + echo "Run transforms/universal/ededup completed" + + test-kfp-v2: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/ededup workflow-test + header_text "Run transforms/universal/ededup completed" diff --git a/.github/workflows/test-universal-fdedup-kfp.yml b/.github/workflows/test-universal-fdedup-kfp.yml new file mode 100644 index 000000000..9d331ed47 --- /dev/null +++ b/.github/workflows/test-universal-fdedup-kfp.yml @@ -0,0 +1,114 @@ +# +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files +# +name: Test KFP - transforms/universal/fdedup + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/universal/fdedup/**" + - "!kfp/**" # This is tested in separate workflow + - "!data-processing-lib/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/universal/fdedup/**" + - "!data-processing-lib/**" # This is tested in separate workflow + - "!kfp/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + + +jobs: + test-kfp-v1: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/fdedup workflow-test + echo "Run transforms/universal/fdedup completed" + + test-kfp-v2: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/fdedup workflow-test + header_text "Run transforms/universal/fdedup completed" diff --git a/.github/workflows/test-universal-filter-kfp.yml b/.github/workflows/test-universal-filter-kfp.yml new file mode 100644 index 000000000..59ebbde3c --- /dev/null +++ b/.github/workflows/test-universal-filter-kfp.yml @@ -0,0 +1,114 @@ +# +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files +# +name: Test KFP - transforms/universal/filter + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/universal/filter/**" + - "!kfp/**" # This is tested in separate workflow + - "!data-processing-lib/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/universal/filter/**" + - "!data-processing-lib/**" # This is tested in separate workflow + - "!kfp/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + + +jobs: + test-kfp-v1: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/filter workflow-test + echo "Run transforms/universal/filter completed" + + test-kfp-v2: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/filter workflow-test + header_text "Run transforms/universal/filter completed" diff --git a/.github/workflows/test-universal-noop-kfp.yml b/.github/workflows/test-universal-noop-kfp.yml new file mode 100644 index 000000000..19c62ab49 --- /dev/null +++ b/.github/workflows/test-universal-noop-kfp.yml @@ -0,0 +1,114 @@ +# +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files +# +name: Test KFP - transforms/universal/noop + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/universal/noop/**" + - "!kfp/**" # This is tested in separate workflow + - "!data-processing-lib/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/universal/noop/**" + - "!data-processing-lib/**" # This is tested in separate workflow + - "!kfp/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + + +jobs: + test-kfp-v1: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/noop workflow-test + echo "Run transforms/universal/noop completed" + + test-kfp-v2: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/noop workflow-test + header_text "Run transforms/universal/noop completed" diff --git a/.github/workflows/test-universal-profiler-kfp.yml b/.github/workflows/test-universal-profiler-kfp.yml new file mode 100644 index 000000000..3d377922d --- /dev/null +++ b/.github/workflows/test-universal-profiler-kfp.yml @@ -0,0 +1,114 @@ +# +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files +# +name: Test KFP - transforms/universal/profiler + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/universal/profiler/**" + - "!kfp/**" # This is tested in separate workflow + - "!data-processing-lib/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/universal/profiler/**" + - "!data-processing-lib/**" # This is tested in separate workflow + - "!kfp/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + + +jobs: + test-kfp-v1: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/profiler workflow-test + echo "Run transforms/universal/profiler completed" + + test-kfp-v2: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/profiler workflow-test + header_text "Run transforms/universal/profiler completed" diff --git a/.github/workflows/test-universal-resize-kfp.yml b/.github/workflows/test-universal-resize-kfp.yml new file mode 100644 index 000000000..fe7377178 --- /dev/null +++ b/.github/workflows/test-universal-resize-kfp.yml @@ -0,0 +1,114 @@ +# +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files +# +name: Test KFP - transforms/universal/resize + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/universal/resize/**" + - "!kfp/**" # This is tested in separate workflow + - "!data-processing-lib/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/universal/resize/**" + - "!data-processing-lib/**" # This is tested in separate workflow + - "!kfp/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + + +jobs: + test-kfp-v1: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/resize workflow-test + echo "Run transforms/universal/resize completed" + + test-kfp-v2: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/resize workflow-test + header_text "Run transforms/universal/resize completed" diff --git a/.github/workflows/test-universal-tokenization-kfp.yml b/.github/workflows/test-universal-tokenization-kfp.yml new file mode 100644 index 000000000..f127db59b --- /dev/null +++ b/.github/workflows/test-universal-tokenization-kfp.yml @@ -0,0 +1,114 @@ +# +# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files +# +name: Test KFP - transforms/universal/tokenization + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths: + - "transforms/universal/tokenization/**" + - "!kfp/**" # This is tested in separate workflow + - "!data-processing-lib/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "transforms/universal/tokenization/**" + - "!data-processing-lib/**" # This is tested in separate workflow + - "!kfp/**" # This is tested in separate workflow + - "!**.md" + - "!**/doc/**" + - "!**/images/**" + - "!**.gitignore" + + +jobs: + test-kfp-v1: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/tokenization workflow-test + echo "Run transforms/universal/tokenization completed" + + test-kfp-v2: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + make -C transforms/universal/tokenization workflow-test + header_text "Run transforms/universal/tokenization completed" diff --git a/.github/workflows/test.yml.old b/.github/workflows/test.yml.old deleted file mode 100644 index 552ac012b..000000000 --- a/.github/workflows/test.yml.old +++ /dev/null @@ -1,405 +0,0 @@ -name: Build, Test and (Optionally) Push images - -on: - workflow_dispatch: - push: - branches: - - "dev" - - "releases/**" - tags: - - "*" - paths-ignore: - - "**.md" - - "examples/**" - - "**/doc/**" - - "**/.gitignore" - - "**/.dockerignore" - pull_request: - branches: - - "dev" - - "releases/**" - paths-ignore: - - "**.md" - - "examples/**" - - "**/doc/**" - - "**/.gitignore" - - "**/.dockerignore" -env: - KFP_BLACK_LIST: "doc_chunk-ray,pdf2parquet-ray,pii_redactor" - -jobs: - check_if_push_images: - # check whether the Docker images should be pushed to the remote repository - # The images are pushed if it is a merge to dev branch or a new tag is created. - # The latter being part of the release process. - # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. - runs-on: ubuntu-22.04 - outputs: - publish_images: ${{ steps.version.outputs.publish_images }} - steps: - - id: version - run: | - publish_images='false' - if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; - then - publish_images='true' - fi - if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; - then - publish_images='true' - fi - echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" - test-make: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Test top-level recursive make targets. - run: | - make -n clean test build publish set-versions - test-python-lib: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Test data-processing-lib/python - run: | - make -C data-processing-lib/python DOCKER=docker venv test - test-ray-lib: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Test data-processing-lib/ray - run: | - make -C data-processing-lib/ray DOCKER=docker venv test - test-spark-lib: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Test data-processing-lib/spark - run: | - make -C data-processing-lib/spark DOCKER=docker venv test - test-code: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Test Code Transforms - run: | - make -C transforms/code DOCKER=docker test-src - test-language: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Test Language Transforms - run: | - make -C transforms/language DOCKER=docker test-src - test-universal: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Test Universal Transforms - run: | - make -C transforms/universal DOCKER=docker test-src - test-tools: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Test tools - run: | - make -C tools DOCKER=docker venv test - test-kfp-v1: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Test KFP libs (shared and v1) and run a workflow - timeout-minutes: 120 - run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - while : - do - dir=("code" "universal" "language") && index=$(($RANDOM % ${#dir[@]})) && subdirs=${dir[$index]} && transforms=($(find transforms/$subdirs -type d -maxdepth 1 -mindepth 1 )) - set -- "${transforms[@]}" && transforms=("$@") && size=${#transforms[@]} && index=$(($RANDOM % $size)) - transform=$(basename "${transforms[$index]}") - if [ -d ${transforms[$index]}/kfp_ray ] && echo ${KFP_BLACK_LIST} | grep -qv ${transform} ; then - header_text "Running ${transforms[$index]} workflow test" - break - fi - done - make -C ${transforms[$index]} workflow-test - echo "Run ${transforms[$index]} completed" - - test-kfp-v2: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Test KFP libs (shared and v2) and run a workflow - timeout-minutes: 120 - run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - while : - do - dir=("code" "universal" "language") && index=$(($RANDOM % ${#dir[@]})) && subdirs=${dir[$index]} && transforms=($(find transforms/$subdirs -type d -maxdepth 1 -mindepth 1 )) - set -- "${transforms[@]}" && transforms=("$@") && size=${#transforms[@]} && index=$(($RANDOM % $size)) - transform=$(basename "${transforms[$index]}") - if [ -d ${transforms[$index]}/kfp_ray ] && echo ${KFP_BLACK_LIST} | grep -qv ${transform} ; then - header_text "Running ${transforms[$index]} workflow test" - break - fi - done - make -C ${transforms[$index]} workflow-test - header_text "Run ${transforms[$index]} completed" - test-data-processing-lib-images: - needs: [check_if_push_images] - if: needs.check_if_push_images.outputs.publish_images == 'true' - runs-on: ubuntu-22.04 - env: - DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} - DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} - timeout-minutes: 30 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Test Code Transform Images - run: | - make -C data-processing-lib/spark DOCKER=docker image - - name: - Print space - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - docker images - - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' - run: | - make -C data-processing-lib/spark publish-image - test-code-images: - needs: [check_if_push_images] - runs-on: ubuntu-22.04 - timeout-minutes: 30 - env: - DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} - DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Test Code Transform Images - run: | - make -C data-processing-lib DOCKER=docker image - make -C transforms/code DOCKER=docker test-image - - name: - Print space - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - docker images - - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' - run: | - make -C transforms/code publish - test-language-images: - needs: [check_if_push_images] - runs-on: ubuntu-22.04 - timeout-minutes: 120 - env: - DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} - DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Test language Transform Images - run: | - make -C data-processing-lib DOCKER=docker image - make -C transforms/language DOCKER=docker test-image - - name: Print space - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: df -h - - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' - run: make -C transforms/language publish - - test-universal-images: - needs: [check_if_push_images] - runs-on: ubuntu-22.04 - timeout-minutes: 120 - env: - DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} - DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Test Universal Transform Images - run: | - make -C data-processing-lib/spark DOCKER=docker image - make -C transforms/universal DOCKER=docker test-image - - name: - Print space - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - docker images - - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' - run: make -C transforms/universal publish - build-kfp-components: - needs: [check_if_push_images] - runs-on: ubuntu-22.04 - timeout-minutes: 30 - env: - DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} - DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Build - run: | - make -C kfp/kfp_ray_components DOCKER=docker image - make KFPv2=1 -C kfp/kfp_ray_components DOCKER=docker image - - name: Publish images - if: needs.check_if_push_images.outputs.publish_images == 'true' - run: make -C kfp/kfp_ray_components publish - test-tool-images: - runs-on: ubuntu-22.04 - timeout-minutes: 30 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Build and Test Tool images - run: | - make -C tools/ingest2parquet DOCKER=docker test-image - diff --git a/data-processing-lib/ray/README.md b/data-processing-lib/ray/README.md index 74f8d4ae8..dd623698b 100644 --- a/data-processing-lib/ray/README.md +++ b/data-processing-lib/ray/README.md @@ -1,5 +1,5 @@ # Data Processing Library -This provides a python framework for developing _transforms_ +This provides a python framework for developing _transforms_99 on data stored in files - currently parquet files are supported - and running them in a [ray](https://www.ray.io/) cluster. Data files may be stored in the local file system or COS/S3. diff --git a/kfp/kfp_ray_components/README.md b/kfp/kfp_ray_components/README.md index 8bc49ede9..4d89d1b50 100644 --- a/kfp/kfp_ray_components/README.md +++ b/kfp/kfp_ray_components/README.md @@ -27,7 +27,7 @@ Once the components are implemented we also implement their interfaces as a comp * The component’s inputs and outputs. * The container image that your component’s code runs in, the command to use to run your component’s code, and the -command-line arguments to pass to your component’s code. +command-line arguments to pass to your component’s code99. * The component’s metadata, such as the name and description. Components specifications are provided here: diff --git a/kfp/kfp_ray_components/src/execute_ray_job.py b/kfp/kfp_ray_components/src/execute_ray_job.py index 7d9c3d98d..c0efa8330 100644 --- a/kfp/kfp_ray_components/src/execute_ray_job.py +++ b/kfp/kfp_ray_components/src/execute_ray_job.py @@ -21,7 +21,7 @@ parser.add_argument("-ap", "--additional_params", type=str, default="{}") parser.add_argument("-su", "--server_url", type=str, default="") # The component converts the dictionary to json string - parser.add_argument("-ep", "--exec_params", type=str, default="{}") + parser.add_argument("-epu", "--exec_params", type=str, default="{}") parser.add_argument("-esn", "--exec_script_name", default="transformer_launcher.py", type=str) args = parser.parse_args() From b7e5d0deb030695c0ab79c674dcebf201c5cdae2 Mon Sep 17 00:00:00 2001 From: Revital Sur Date: Tue, 24 Sep 2024 10:30:30 +0300 Subject: [PATCH 2/6] Minor change. Signed-off-by: Revital Sur --- .github/workflows/test-kfp.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/test-kfp.yml b/.github/workflows/test-kfp.yml index dc8bc2bd5..6719c322e 100644 --- a/.github/workflows/test-kfp.yml +++ b/.github/workflows/test-kfp.yml @@ -10,7 +10,14 @@ on: - "*" paths: - "kfp/**" + - "!kfp/**.md" + - "!kfp/**/test/**" + - "!kfp/**/doc/**" - "data-processing-lib/**" + - "!data-processing-lib/**.md" + - "!data-processing-lib/**/.gitignore" + - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" - "!**.md" - "!**/doc/**" - "!**/images/**" @@ -21,7 +28,15 @@ on: - "releases/**" paths: - "kfp/**" + - "!kfp/**/test/**" + - "!kfp/**.md" + - "!kfp/**/doc/**" - "data-processing-lib/**" + - "!data-processing-lib/**/test/**" + - "!data-processing-lib/**/test-data/**" + - "!data-processing-lib/**.md" + - "!data-processing-lib/**/doc/**" + - "!data-processing-lib/**/.gitignore" - "!**.md" - "!**/doc/**" - "!**/images/**" From 6796e5a318b892f62df126a7b4139738659ada55 Mon Sep 17 00:00:00 2001 From: Revital Sur Date: Wed, 25 Sep 2024 23:20:15 +0300 Subject: [PATCH 3/6] Address review comments. Signed-off-by: Revital Sur --- .github/workflows/Makefile | 2 +- .github/workflows/build-library.yml.old | 59 +++++++++++++++++++ .github/workflows/runs.h | 3 - data-processing-lib/ray/README.md | 2 +- kfp/kfp_ray_components/README.md | 2 +- kfp/kfp_ray_components/src/execute_ray_job.py | 2 +- 6 files changed, 63 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/build-library.yml.old delete mode 100755 .github/workflows/runs.h diff --git a/.github/workflows/Makefile b/.github/workflows/Makefile index 8a17ffa08..275fd4688 100644 --- a/.github/workflows/Makefile +++ b/.github/workflows/Makefile @@ -11,7 +11,7 @@ LANG_TRANSFORMS=doc_chunk doc_quality lang_id pdf2parquet pii_redactor text_enco KFP_BLACK_LIST="doc_chunk,pdf2parquet,pii_redactor" transform-tests: - $(MAKE) TRANSFORM_SUBDIR=universal .transform-tests .transform-kfp-tests + $(MAKE) TRANSFORM_SUBDIR=universal .transform-tests $(MAKE) TRANSFORM_SUBDIR=universal .transform-kfp-tests $(MAKE) TRANSFORM_SUBDIR=language .transform-tests $(MAKE) TRANSFORM_SUBDIR=language .transform-kfp-tests diff --git a/.github/workflows/build-library.yml.old b/.github/workflows/build-library.yml.old new file mode 100644 index 000000000..d7219ea34 --- /dev/null +++ b/.github/workflows/build-library.yml.old @@ -0,0 +1,59 @@ +name: Build Library + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + paths: + - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/.gitignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths: + - "data-processing-lib/**" + - "!**.md" + - "!**/doc/**" + - "!**/.gitignore" +jobs: + build-python-lib: + runs-on: ubuntu-22.04 + strategy: + matrix: + python: + - "3.11" + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build data-processing-lib/python + run: | + make -C data-processing-lib/python DOCKER=docker venv build + build-ray-lib: + runs-on: ubuntu-22.04 + strategy: + matrix: + python: + - "3.11" + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build data-processing-lib/ray + run: | + make -C data-processing-lib/ray DOCKER=docker venv build + build-spark-lib: + runs-on: ubuntu-22.04 + strategy: + matrix: + python: + - "3.11" + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build data-processing-lib/spark + run: | + make -C data-processing-lib/spark DOCKER=docker venv build diff --git a/.github/workflows/runs.h b/.github/workflows/runs.h deleted file mode 100755 index 5a54bb616..000000000 --- a/.github/workflows/runs.h +++ /dev/null @@ -1,3 +0,0 @@ - -echo ${PWD} -find ../../transforms/universal -mindepth 1 -maxdepth 1 -type d diff --git a/data-processing-lib/ray/README.md b/data-processing-lib/ray/README.md index dd623698b..74f8d4ae8 100644 --- a/data-processing-lib/ray/README.md +++ b/data-processing-lib/ray/README.md @@ -1,5 +1,5 @@ # Data Processing Library -This provides a python framework for developing _transforms_99 +This provides a python framework for developing _transforms_ on data stored in files - currently parquet files are supported - and running them in a [ray](https://www.ray.io/) cluster. Data files may be stored in the local file system or COS/S3. diff --git a/kfp/kfp_ray_components/README.md b/kfp/kfp_ray_components/README.md index 4d89d1b50..8bc49ede9 100644 --- a/kfp/kfp_ray_components/README.md +++ b/kfp/kfp_ray_components/README.md @@ -27,7 +27,7 @@ Once the components are implemented we also implement their interfaces as a comp * The component’s inputs and outputs. * The container image that your component’s code runs in, the command to use to run your component’s code, and the -command-line arguments to pass to your component’s code99. +command-line arguments to pass to your component’s code. * The component’s metadata, such as the name and description. Components specifications are provided here: diff --git a/kfp/kfp_ray_components/src/execute_ray_job.py b/kfp/kfp_ray_components/src/execute_ray_job.py index c0efa8330..7d9c3d98d 100644 --- a/kfp/kfp_ray_components/src/execute_ray_job.py +++ b/kfp/kfp_ray_components/src/execute_ray_job.py @@ -21,7 +21,7 @@ parser.add_argument("-ap", "--additional_params", type=str, default="{}") parser.add_argument("-su", "--server_url", type=str, default="") # The component converts the dictionary to json string - parser.add_argument("-epu", "--exec_params", type=str, default="{}") + parser.add_argument("-ep", "--exec_params", type=str, default="{}") parser.add_argument("-esn", "--exec_script_name", default="transformer_launcher.py", type=str) args = parser.parse_args() From 58eecf05d9a5c0e24c9b428e41254dd5d888f960 Mon Sep 17 00:00:00 2001 From: Revital Sur Date: Wed, 25 Sep 2024 23:22:04 +0300 Subject: [PATCH 4/6] Minor change. Signed-off-by: Revital Sur --- .github/workflows/test.yml.old | 405 +++++++++++++++++++++++++++++++++ 1 file changed, 405 insertions(+) create mode 100644 .github/workflows/test.yml.old diff --git a/.github/workflows/test.yml.old b/.github/workflows/test.yml.old new file mode 100644 index 000000000..552ac012b --- /dev/null +++ b/.github/workflows/test.yml.old @@ -0,0 +1,405 @@ +name: Build, Test and (Optionally) Push images + +on: + workflow_dispatch: + push: + branches: + - "dev" + - "releases/**" + tags: + - "*" + paths-ignore: + - "**.md" + - "examples/**" + - "**/doc/**" + - "**/.gitignore" + - "**/.dockerignore" + pull_request: + branches: + - "dev" + - "releases/**" + paths-ignore: + - "**.md" + - "examples/**" + - "**/doc/**" + - "**/.gitignore" + - "**/.dockerignore" +env: + KFP_BLACK_LIST: "doc_chunk-ray,pdf2parquet-ray,pii_redactor" + +jobs: + check_if_push_images: + # check whether the Docker images should be pushed to the remote repository + # The images are pushed if it is a merge to dev branch or a new tag is created. + # The latter being part of the release process. + # The images tag is derived from the value of the DOCKER_IMAGE_VERSION variable set in the .make.versions file. + runs-on: ubuntu-22.04 + outputs: + publish_images: ${{ steps.version.outputs.publish_images }} + steps: + - id: version + run: | + publish_images='false' + if [[ ${GITHUB_REF} == refs/heads/dev && ${GITHUB_EVENT_NAME} != 'pull_request' && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + if [[ ${GITHUB_REF} == refs/tags/* && ${GITHUB_REPOSITORY} == IBM/data-prep-kit ]] ; + then + publish_images='true' + fi + echo "publish_images=$publish_images" >> "$GITHUB_OUTPUT" + test-make: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Test top-level recursive make targets. + run: | + make -n clean test build publish set-versions + test-python-lib: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Test data-processing-lib/python + run: | + make -C data-processing-lib/python DOCKER=docker venv test + test-ray-lib: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Test data-processing-lib/ray + run: | + make -C data-processing-lib/ray DOCKER=docker venv test + test-spark-lib: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Test data-processing-lib/spark + run: | + make -C data-processing-lib/spark DOCKER=docker venv test + test-code: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test Code Transforms + run: | + make -C transforms/code DOCKER=docker test-src + test-language: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test Language Transforms + run: | + make -C transforms/language DOCKER=docker test-src + test-universal: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test Universal Transforms + run: | + make -C transforms/universal DOCKER=docker test-src + test-tools: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Test tools + run: | + make -C tools DOCKER=docker venv test + test-kfp-v1: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v1) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + while : + do + dir=("code" "universal" "language") && index=$(($RANDOM % ${#dir[@]})) && subdirs=${dir[$index]} && transforms=($(find transforms/$subdirs -type d -maxdepth 1 -mindepth 1 )) + set -- "${transforms[@]}" && transforms=("$@") && size=${#transforms[@]} && index=$(($RANDOM % $size)) + transform=$(basename "${transforms[$index]}") + if [ -d ${transforms[$index]}/kfp_ray ] && echo ${KFP_BLACK_LIST} | grep -qv ${transform} ; then + header_text "Running ${transforms[$index]} workflow test" + break + fi + done + make -C ${transforms[$index]} workflow-test + echo "Run ${transforms[$index]} completed" + + test-kfp-v2: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test KFP libs (shared and v2) and run a workflow + timeout-minutes: 120 + run: | + export REPOROOT=$PWD + export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup + source $K8S_SETUP_SCRIPTS/requirements.env + export PATH=$PATH:/tmp/ + curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 + chmod 777 /tmp/kind + curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 + chmod 700 /tmp/get_helm.sh + HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo + chmod 777 /tmp/helm + curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl + chmod 777 /tmp/kubectl + curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc + chmod +x /tmp/mc + export DEPLOY_KUBEFLOW=1 + export KFPv2=1 + make -C $K8S_SETUP_SCRIPTS setup + make -C kfp/kfp_support_lib test + make -C transforms workflow-build + source $K8S_SETUP_SCRIPTS/common.sh + while : + do + dir=("code" "universal" "language") && index=$(($RANDOM % ${#dir[@]})) && subdirs=${dir[$index]} && transforms=($(find transforms/$subdirs -type d -maxdepth 1 -mindepth 1 )) + set -- "${transforms[@]}" && transforms=("$@") && size=${#transforms[@]} && index=$(($RANDOM % $size)) + transform=$(basename "${transforms[$index]}") + if [ -d ${transforms[$index]}/kfp_ray ] && echo ${KFP_BLACK_LIST} | grep -qv ${transform} ; then + header_text "Running ${transforms[$index]} workflow test" + break + fi + done + make -C ${transforms[$index]} workflow-test + header_text "Run ${transforms[$index]} completed" + test-data-processing-lib-images: + needs: [check_if_push_images] + if: needs.check_if_push_images.outputs.publish_images == 'true' + runs-on: ubuntu-22.04 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test Code Transform Images + run: | + make -C data-processing-lib/spark DOCKER=docker image + - name: + Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: | + make -C data-processing-lib/spark publish-image + test-code-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 30 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test Code Transform Images + run: | + make -C data-processing-lib DOCKER=docker image + make -C transforms/code DOCKER=docker test-image + - name: + Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: | + make -C transforms/code publish + test-language-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test language Transform Images + run: | + make -C data-processing-lib DOCKER=docker image + make -C transforms/language DOCKER=docker test-image + - name: Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: df -h + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C transforms/language publish + + test-universal-images: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 120 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Test Universal Transform Images + run: | + make -C data-processing-lib/spark DOCKER=docker image + make -C transforms/universal DOCKER=docker test-image + - name: + Print space + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + docker images + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C transforms/universal publish + build-kfp-components: + needs: [check_if_push_images] + runs-on: ubuntu-22.04 + timeout-minutes: 30 + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_KEY: ${{ secrets.DOCKER_REGISTRY_KEY }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Free up space in github runner + # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 + run: | + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup + sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true + df -h + - name: Build + run: | + make -C kfp/kfp_ray_components DOCKER=docker image + make KFPv2=1 -C kfp/kfp_ray_components DOCKER=docker image + - name: Publish images + if: needs.check_if_push_images.outputs.publish_images == 'true' + run: make -C kfp/kfp_ray_components publish + test-tool-images: + runs-on: ubuntu-22.04 + timeout-minutes: 30 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build and Test Tool images + run: | + make -C tools/ingest2parquet DOCKER=docker test-image + From 7fe9c87fb665bbae05ffb54985196cb4228ad198 Mon Sep 17 00:00:00 2001 From: Revital Sur Date: Sun, 29 Sep 2024 10:41:06 +0300 Subject: [PATCH 5/6] Remove test-code-inputcode2parquet.yml Signed-off-by: Revital Sur --- .../test-code-inputcode2parquet-kfp.yml | 114 ------------------ 1 file changed, 114 deletions(-) delete mode 100644 .github/workflows/test-code-inputcode2parquet-kfp.yml diff --git a/.github/workflows/test-code-inputcode2parquet-kfp.yml b/.github/workflows/test-code-inputcode2parquet-kfp.yml deleted file mode 100644 index 32ed1ad07..000000000 --- a/.github/workflows/test-code-inputcode2parquet-kfp.yml +++ /dev/null @@ -1,114 +0,0 @@ -# -# DO NOT EDIT THIS FILE: it is generated from test-transform.template, Edit there and run make to change these files -# -name: Test KFP - transforms/code/inputcode2parquet - -on: - workflow_dispatch: - push: - branches: - - "dev" - - "releases/**" - tags: - - "*" - paths: - - "transforms/code/inputcode2parquet/**" - - "!kfp/**" # This is tested in separate workflow - - "!data-processing-lib/**" # This is tested in separate workflow - - "!**.md" - - "!**/doc/**" - - "!**/images/**" - - "!**.gitignore" - pull_request: - branches: - - "dev" - - "releases/**" - paths: - - "transforms/code/inputcode2parquet/**" - - "!data-processing-lib/**" # This is tested in separate workflow - - "!kfp/**" # This is tested in separate workflow - - "!**.md" - - "!**/doc/**" - - "!**/images/**" - - "!**.gitignore" - - -jobs: - test-kfp-v1: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Test KFP libs (shared and v1) and run a workflow - timeout-minutes: 120 - run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/inputcode2parquet workflow-test - echo "Run transforms/code/inputcode2parquet completed" - - test-kfp-v2: - runs-on: ubuntu-22.04 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Free up space in github runner - # Free space as indicated here : https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 - run: | - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup - sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true - df -h - - name: Test KFP libs (shared and v2) and run a workflow - timeout-minutes: 120 - run: | - export REPOROOT=$PWD - export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup - source $K8S_SETUP_SCRIPTS/requirements.env - export PATH=$PATH:/tmp/ - curl -Lo /tmp/kind https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64 - chmod 777 /tmp/kind - curl -fsSL -o /tmp/get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 - chmod 700 /tmp/get_helm.sh - HELM_INSTALL_DIR=/tmp/ /tmp/get_helm.sh -v v${HELM_VERSION} --no-sudo - chmod 777 /tmp/helm - curl -L https://dl.k8s.io/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl -o /tmp/kubectl - chmod 777 /tmp/kubectl - curl https://dl.min.io/client/mc/release/linux-amd64/mc --create-dirs -o /tmp/mc - chmod +x /tmp/mc - export DEPLOY_KUBEFLOW=1 - export KFPv2=1 - make -C $K8S_SETUP_SCRIPTS setup - make -C kfp/kfp_support_lib test - make -C transforms workflow-build - source $K8S_SETUP_SCRIPTS/common.sh - make -C transforms/code/inputcode2parquet workflow-test - header_text "Run transforms/code/inputcode2parquet completed" From 48575f8839e215e4b881f61baafb22a2152686e4 Mon Sep 17 00:00:00 2001 From: Revital Sur Date: Sun, 29 Sep 2024 10:44:03 +0300 Subject: [PATCH 6/6] Addres review comments. Signed-off-by: Revital Sur --- .github/workflows/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index d4a09b1c3..85fa055a3 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -9,7 +9,6 @@ Includes python, ray, spark and kfp_ray as available. 2. When the core dpk lib components files changes, test all transforms 3. When the shared kfp components changes or core dpk lib components files changes, test a randomly selected transform test. Otherwise run kfp test for the changed transforms. -4. Extra credit: If .md or other non-code changes are made, run no tests. #### Assumptions 1. All transforms will have test workflows. A transform can disable its tests locally