Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix kfp workflows which were created with a bad template #672

Merged
merged 10 commits into from
Oct 7, 2024
Merged

Fix kfp workflows which were created with a bad template #672

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
ae47d8a
fix typo in kfp workflows
daw3rd Oct 4, 2024
9a6a704
fix another typo in kfp workflows
daw3rd Oct 4, 2024
aef92ae
innocuous change in .make.workflows to trigger kfp tests
daw3rd Oct 4, 2024
3f52362
fix missing path in workflow templates
daw3rd Oct 4, 2024
78d4fd9
fix repo_level_order_wf.py default float value of str type
daw3rd Oct 5, 2024
ff8f185
workflow generation fixes and kfp workflow to test for kfp_ray/Makefile
daw3rd Oct 5, 2024
f710d3f
disable kfp cicd for repo_level_ordering and text_encoder
daw3rd Oct 5, 2024
acd34a8
fix testing for makefile in kfp workflows
daw3rd Oct 5, 2024
822841a
remove kfp_support_list testing from transform's kfp git workflow
daw3rd Oct 5, 2024
3704f2a
disable kfp cicd on license_select
daw3rd Oct 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 16 additions & 7 deletions .github/workflows/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,39 +10,48 @@ LANG_TRANSFORMS=doc_chunk doc_quality lang_id pdf2parquet pii_redactor text_enco
# A list that holds transforms that should not be tested with KFP

transform-tests:
-rm -rf test-universal-*.yml
$(MAKE) TRANSFORM_SUBDIR=universal .transform-tests
$(MAKE) TRANSFORM_SUBDIR=universal .transform-kfp-tests
-rm -rf test-language-*.yml
$(MAKE) TRANSFORM_SUBDIR=language .transform-tests
$(MAKE) TRANSFORM_SUBDIR=language .transform-kfp-tests
-rm -rf test-code-*.yml
$(MAKE) TRANSFORM_SUBDIR=code .transform-tests
$(MAKE) TRANSFORM_SUBDIR=code .transform-kfp-tests

# Expects
# TRANSFORM_SUBDIR transforms subdirectory (such as universal)
.transform-tests:
@for i in $$(find ../../transforms/$(TRANSFORM_SUBDIR) -mindepth 1 -maxdepth 1 -type d); do \
dir=$$(basename $$i); \
yml=test-$(TRANSFORM_SUBDIR)-$$dir.yml; \
echo Generating $$yml; \
cat test-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/$${TRANSFORM_SUBDIR}/$$dir?g" > $$yml; \
z=$$(ls $$i/Makefile*); \
if [ ! -z "$$z" ]; then \
dir=$$(basename $$i); \
yml=test-$(TRANSFORM_SUBDIR)-$$dir.yml; \
echo Generating $$yml; \
cat test-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/$${TRANSFORM_SUBDIR}/$$dir?g" > $$yml; \
fi; \
done

.transform-kfp-tests:
@KFP_BLACK_LIST=$$(cd ../..; bash scripts/check-workflows.sh -show-kfp-black-list); \
for i in $$(find ../../transforms/$(TRANSFORM_SUBDIR) -mindepth 1 -maxdepth 1 -type d); do \
z=$$(ls $$i/Makefile*); \
if [ ! -z "$$z" ]; then \
dir=$$(basename $$i); \
yml=test-$(TRANSFORM_SUBDIR)-$$dir-kfp.yml; \
if [ ! -d ../../transforms/$(TRANSFORM_SUBDIR)/$$dir/kfp_ray ]; then \
echo No kfp_ray directory for $$dir. Skipping generation of $$yml; \
echo No kfp_ray directory for $$dir. Skipping generation of $$yml; \
continue; \
fi; \
z=$$(echo $${KFP_BLACK_LIST} | grep $$dir); \
if [ ! -z "$$z" ]; then \
echo $$dir is black listed. Skipping generation of $$yml; \
echo $$dir is black listed. Skipping generation of $$yml; \
continue; \
fi; \
echo Generating $$yml; \
echo Generating $$yml; \
cat test-kfp-transform.template | sed -e "s?@TARGET_TRANSFORM_DIR@?transforms/$${TRANSFORM_SUBDIR}/$$dir?g" > $$yml; \
fi; \
done


Expand Down
20 changes: 10 additions & 10 deletions .github/workflows/test-code-code2parquet-kfp.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ on:
- "*"
paths:
- ".make.*"
- "transforms/.make.workflow"
- "transforms/.make.workflows"
- "transforms/code/code2parquet/**"
- "!kfp/**" # This is tested in separate workflow
- "!data-processing-lib/**" # This is tested in separate workflow
Expand All @@ -27,7 +27,7 @@ on:
- "releases/**"
paths:
- ".make.*"
- "transforms/.make.workflow"
- "transforms/.make.workflows"
- "transforms/code/code2parquet/**"
- "!data-processing-lib/**" # This is tested in separate workflow
- "!kfp/**" # This is tested in separate workflow
Expand Down Expand Up @@ -56,10 +56,10 @@ jobs:
sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
df -h
- name: Test KFP libs (shared and v1) and run a workflow
- name: Test V1 KFP workflow for transforms/code/code2parquet
timeout-minutes: 120
run: |
if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then
if [ -e "transforms/code/code2parquet/Makefile" -a -e "transforms/code/code2parquet/kfp_ray/Makefile" ]; then
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can we combine the lines 62-86 in a shared Make rule, which will be the same for all transformers, and just change its arguments?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

100% agree with this, but can I put that on @revit13 for another PR.

export REPOROOT=$PWD
export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
source $K8S_SETUP_SCRIPTS/requirements.env
Expand All @@ -76,13 +76,13 @@ jobs:
chmod +x /tmp/mc
export DEPLOY_KUBEFLOW=1
make -C $K8S_SETUP_SCRIPTS setup
make -C kfp/kfp_support_lib test
#make -C kfp/kfp_support_lib test
make -C transforms/code/code2parquet workflow-build
source $K8S_SETUP_SCRIPTS/common.sh
make -C transforms/code/code2parquet workflow-test
echo "Run transforms/code/code2parquet completed"
else
echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray"
echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray/Makefile"
fi

test-kfp-v2:
Expand All @@ -99,10 +99,10 @@ jobs:
sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
df -h
- name: Test KFP libs (shared and v2) and run a workflow
- name: Test V2 KFP workflow for transforms/code/code2parquet
timeout-minutes: 120
run: |
if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then
if [ -e "transforms/code/code2parquet/Makefile" -a -e "transforms/code/code2parquet/kfp_ray/Makefile" ]; then
export REPOROOT=$PWD
export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
source $K8S_SETUP_SCRIPTS/requirements.env
Expand All @@ -120,11 +120,11 @@ jobs:
export DEPLOY_KUBEFLOW=1
export KFPv2=1
make -C $K8S_SETUP_SCRIPTS setup
make -C kfp/kfp_support_lib test
#make -C kfp/kfp_support_lib test
make -C transforms/code/code2parquet workflow-build
source $K8S_SETUP_SCRIPTS/common.sh
make -C transforms/code/code2parquet workflow-test
echo "Run transforms/code/code2parquet completed"
else
echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray"
echo "Skipping transforms/code/code2parquet kfp test for lack of Makefile and/or kfp_ray/Makefile"
fi
20 changes: 10 additions & 10 deletions .github/workflows/test-code-code_quality-kfp.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ on:
- "*"
paths:
- ".make.*"
- "transforms/.make.workflow"
- "transforms/.make.workflows"
- "transforms/code/code_quality/**"
- "!kfp/**" # This is tested in separate workflow
- "!data-processing-lib/**" # This is tested in separate workflow
Expand All @@ -27,7 +27,7 @@ on:
- "releases/**"
paths:
- ".make.*"
- "transforms/.make.workflow"
- "transforms/.make.workflows"
- "transforms/code/code_quality/**"
- "!data-processing-lib/**" # This is tested in separate workflow
- "!kfp/**" # This is tested in separate workflow
Expand Down Expand Up @@ -56,10 +56,10 @@ jobs:
sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
df -h
- name: Test KFP libs (shared and v1) and run a workflow
- name: Test V1 KFP workflow for transforms/code/code_quality
timeout-minutes: 120
run: |
if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then
if [ -e "transforms/code/code_quality/Makefile" -a -e "transforms/code/code_quality/kfp_ray/Makefile" ]; then
export REPOROOT=$PWD
export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
source $K8S_SETUP_SCRIPTS/requirements.env
Expand All @@ -76,13 +76,13 @@ jobs:
chmod +x /tmp/mc
export DEPLOY_KUBEFLOW=1
make -C $K8S_SETUP_SCRIPTS setup
make -C kfp/kfp_support_lib test
#make -C kfp/kfp_support_lib test
make -C transforms/code/code_quality workflow-build
source $K8S_SETUP_SCRIPTS/common.sh
make -C transforms/code/code_quality workflow-test
echo "Run transforms/code/code_quality completed"
else
echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray"
echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray/Makefile"
fi

test-kfp-v2:
Expand All @@ -99,10 +99,10 @@ jobs:
sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
df -h
- name: Test KFP libs (shared and v2) and run a workflow
- name: Test V2 KFP workflow for transforms/code/code_quality
timeout-minutes: 120
run: |
if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then
if [ -e "transforms/code/code_quality/Makefile" -a -e "transforms/code/code_quality/kfp_ray/Makefile" ]; then
export REPOROOT=$PWD
export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
source $K8S_SETUP_SCRIPTS/requirements.env
Expand All @@ -120,11 +120,11 @@ jobs:
export DEPLOY_KUBEFLOW=1
export KFPv2=1
make -C $K8S_SETUP_SCRIPTS setup
make -C kfp/kfp_support_lib test
#make -C kfp/kfp_support_lib test
make -C transforms/code/code_quality workflow-build
source $K8S_SETUP_SCRIPTS/common.sh
make -C transforms/code/code_quality workflow-test
echo "Run transforms/code/code_quality completed"
else
echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray"
echo "Skipping transforms/code/code_quality kfp test for lack of Makefile and/or kfp_ray/Makefile"
fi
20 changes: 10 additions & 10 deletions .github/workflows/test-code-header_cleanser-kfp.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ on:
- "*"
paths:
- ".make.*"
- "transforms/.make.workflow"
- "transforms/.make.workflows"
- "transforms/code/header_cleanser/**"
- "!kfp/**" # This is tested in separate workflow
- "!data-processing-lib/**" # This is tested in separate workflow
Expand All @@ -27,7 +27,7 @@ on:
- "releases/**"
paths:
- ".make.*"
- "transforms/.make.workflow"
- "transforms/.make.workflows"
- "transforms/code/header_cleanser/**"
- "!data-processing-lib/**" # This is tested in separate workflow
- "!kfp/**" # This is tested in separate workflow
Expand Down Expand Up @@ -56,10 +56,10 @@ jobs:
sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
df -h
- name: Test KFP libs (shared and v1) and run a workflow
- name: Test V1 KFP workflow for transforms/code/header_cleanser
timeout-minutes: 120
run: |
if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then
if [ -e "transforms/code/header_cleanser/Makefile" -a -e "transforms/code/header_cleanser/kfp_ray/Makefile" ]; then
export REPOROOT=$PWD
export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
source $K8S_SETUP_SCRIPTS/requirements.env
Expand All @@ -76,13 +76,13 @@ jobs:
chmod +x /tmp/mc
export DEPLOY_KUBEFLOW=1
make -C $K8S_SETUP_SCRIPTS setup
make -C kfp/kfp_support_lib test
#make -C kfp/kfp_support_lib test
make -C transforms/code/header_cleanser workflow-build
source $K8S_SETUP_SCRIPTS/common.sh
make -C transforms/code/header_cleanser workflow-test
echo "Run transforms/code/header_cleanser completed"
else
echo "Skipping transforms/code/header_cleanser kfp test for lack of Makefile and/or kfp_ray"
echo "Skipping transforms/code/header_cleanser kfp test for lack of Makefile and/or kfp_ray/Makefile"
fi

test-kfp-v2:
Expand All @@ -99,10 +99,10 @@ jobs:
sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
df -h
- name: Test KFP libs (shared and v2) and run a workflow
- name: Test V2 KFP workflow for transforms/code/header_cleanser
timeout-minutes: 120
run: |
if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then
if [ -e "transforms/code/header_cleanser/Makefile" -a -e "transforms/code/header_cleanser/kfp_ray/Makefile" ]; then
export REPOROOT=$PWD
export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
source $K8S_SETUP_SCRIPTS/requirements.env
Expand All @@ -120,11 +120,11 @@ jobs:
export DEPLOY_KUBEFLOW=1
export KFPv2=1
make -C $K8S_SETUP_SCRIPTS setup
make -C kfp/kfp_support_lib test
#make -C kfp/kfp_support_lib test
make -C transforms/code/header_cleanser workflow-build
source $K8S_SETUP_SCRIPTS/common.sh
make -C transforms/code/header_cleanser workflow-test
echo "Run transforms/code/header_cleanser completed"
else
echo "Skipping transforms/code/header_cleanser kfp test for lack of Makefile and/or kfp_ray"
echo "Skipping transforms/code/header_cleanser kfp test for lack of Makefile and/or kfp_ray/Makefile"
fi
20 changes: 10 additions & 10 deletions .github/workflows/test-code-license_select-kfp.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ on:
- "*"
paths:
- ".make.*"
- "transforms/.make.workflow"
- "transforms/.make.workflows"
- "transforms/code/license_select/**"
- "!kfp/**" # This is tested in separate workflow
- "!data-processing-lib/**" # This is tested in separate workflow
Expand All @@ -27,7 +27,7 @@ on:
- "releases/**"
paths:
- ".make.*"
- "transforms/.make.workflow"
- "transforms/.make.workflows"
- "transforms/code/license_select/**"
- "!data-processing-lib/**" # This is tested in separate workflow
- "!kfp/**" # This is tested in separate workflow
Expand Down Expand Up @@ -56,10 +56,10 @@ jobs:
sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
df -h
- name: Test KFP libs (shared and v1) and run a workflow
- name: Test V1 KFP workflow for transforms/code/license_select
timeout-minutes: 120
run: |
if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then
if [ -e "transforms/code/license_select/Makefile" -a -e "transforms/code/license_select/kfp_ray/Makefile" ]; then
export REPOROOT=$PWD
export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
source $K8S_SETUP_SCRIPTS/requirements.env
Expand All @@ -76,13 +76,13 @@ jobs:
chmod +x /tmp/mc
export DEPLOY_KUBEFLOW=1
make -C $K8S_SETUP_SCRIPTS setup
make -C kfp/kfp_support_lib test
#make -C kfp/kfp_support_lib test
make -C transforms/code/license_select workflow-build
source $K8S_SETUP_SCRIPTS/common.sh
make -C transforms/code/license_select workflow-test
echo "Run transforms/code/license_select completed"
else
echo "Skipping transforms/code/license_select kfp test for lack of Makefile and/or kfp_ray"
echo "Skipping transforms/code/license_select kfp test for lack of Makefile and/or kfp_ray/Makefile"
fi

test-kfp-v2:
Expand All @@ -99,10 +99,10 @@ jobs:
sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /usr/local/share/powershell /usr/share/swift /usr/lib/jvm /usr/local/.ghcup
sudo docker rmi $(docker image ls -aq) >/dev/null 2>&1 || true
df -h
- name: Test KFP libs (shared and v2) and run a workflow
- name: Test V2 KFP workflow for transforms/code/license_select
timeout-minutes: 120
run: |
if [ -e "@TARGET_TRANSFORM_DIR/Makefile" -a -d "@TARGET_TRANSFORM_DIR/kfp_ray" ]; then
if [ -e "transforms/code/license_select/Makefile" -a -e "transforms/code/license_select/kfp_ray/Makefile" ]; then
export REPOROOT=$PWD
export K8S_SETUP_SCRIPTS=$PWD/scripts/k8s-setup
source $K8S_SETUP_SCRIPTS/requirements.env
Expand All @@ -120,11 +120,11 @@ jobs:
export DEPLOY_KUBEFLOW=1
export KFPv2=1
make -C $K8S_SETUP_SCRIPTS setup
make -C kfp/kfp_support_lib test
#make -C kfp/kfp_support_lib test
make -C transforms/code/license_select workflow-build
source $K8S_SETUP_SCRIPTS/common.sh
make -C transforms/code/license_select workflow-test
echo "Run transforms/code/license_select completed"
else
echo "Skipping transforms/code/license_select kfp test for lack of Makefile and/or kfp_ray"
echo "Skipping transforms/code/license_select kfp test for lack of Makefile and/or kfp_ray/Makefile"
fi
Loading